o
    ;i                     @   s2  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZmZmZmZmZmZ d dl m!Z! d d	l"m#Z# d d
l$m%Z% d dl&Z'd dl(Z)d dl&m*Z*m+Z+m,Z,m-Z-m.Z. d dl)m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z: ddl;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZB ddlCmDZDmEZEmFZFmGZGmHZHmIZI ddlJmKZKmLZL ddlMmNZN ddlOmPZP ddlQmRZR ddlSmTZTmUZU ddlVmWZW ddlXmYZY dZZde[de[de\ddfdd Z]G d!d" d"ej^Z_ed#d$G d%d& d&Z`eG d'd( d(ZaG d)d* d*ZbeBebZcG d+d, d,e5d-d.ZdeBedZeG d/d0 d0ZfeBefZgG d1d2 d2efZheBehZiee jjgeeF f ZkG d3d4 d4efZleBelZmd5eneF d6end7eoepepf d8e[d9ed:ef f
d;d<Zqd=e\dee\ fd>d?ZrdS )@    N)AsyncGeneratorAsyncIterator	GeneratorSequence)	dataclass)datetime)BytesIO)PathPurePosixPath)Any	AwaitableBinaryIOCallableOptionalUnion)Message)classproperty)asynccontextmanager)AlreadyExistsErrorConflictErrorInvalidErrorNotFoundErrorVolumeUploadTimeoutError)api_pb2   )LoadContext) EPHEMERAL_OBJECT_HEARTBEAT_SLEEP_get_environment_name_Objectlive_methodlive_method_contextmanagerlive_method_gen)Resolver)TaskContextaclosing	async_mapasync_map_orderedasyncnullcontextretrysynchronize_api)
BLOCK_SIZEFileUploadSpecFileUploadSpec2blob_upload_file!get_file_upload_spec_from_fileobjget_file_upload_spec_from_path)deprecation_warningwarn_if_passing_namespace)Retry)ClientSessionRegistry)check_object_name)as_timestamptimestamp_to_localized_dt)_Client)loggeri  requested_versionactual_versionvolume_namereturnc                 C   sJ   dt dt fdd}|| }||}||kr#td| d| d| dd	S )
zHValidate that the returned volume version matches the requested version.vr<   c                 S   s   | d dt jjfv rt jjS | S )Nr   )r   VolumeFsVersionVOLUME_FS_VERSION_UNSPECIFIEDVOLUME_FS_VERSION_V1)r=    rA   @/home/ubuntu/.local/lib/python3.10/site-packages/modal/volume.py	normalizeT   s   z+_validate_volume_version.<locals>.normalizezVolume 'z' exists but has version vz, not vzc as requested. To access this Volume, either omit the version parameter or use the correct version.N)intr   )r9   r:   r;   rC   n_requestedn_actualrA   rA   rB   _validate_volume_versionM   s   rG   c                   @   s(   e Zd ZdZdZdZdZdZdZdZ	dS )	FileEntryTypez0Type of a file entry listed from a Modal volume.r   r               N)
__name__
__module____qualname____doc__UNSPECIFIEDFILE	DIRECTORYSYMLINKFIFOSOCKETrA   rA   rA   rB   rH   c   s    rH   T)frozenc                   @   sJ   e Zd ZU dZeed< eed< eed< eed< ede	j
dd fdd	Zd
S )	FileEntryz5A file or directory entry listed from a Modal volume.pathtypemtimesizeprotor<   c                 C   s   | |j t|j|j|jdS )N)rY   rZ   r[   r\   )rY   rH   rZ   r[   r\   )clsr]   rA   rA   rB   _from_protow   s   zFileEntry._from_protoN)rM   rN   rO   rP   str__annotations__rH   rD   classmethodr   rX   r_   rA   rA   rA   rB   rX   n   s   
 rX   c                   @   s2   e Zd ZU dZee ed< eed< ee ed< dS )
VolumeInfoz$Information about the Volume object.name
created_at
created_byN)rM   rN   rO   rP   r   r`   ra   r   rA   rA   rA   rB   rc      s
   
 rc   c                   @   s   e Zd ZdZeddddddedee dedee d	ee	 d
dfddZ
eddddddee deeeef  ded	ee	 d
ejd f
ddZedddddededee d	ee	 fddZdS )_VolumeManagerz9Namespace with methods for managing named Volume objects.NF)versionallow_existingenvironment_nameclientrd   rh   ri   rj   rk   r<   c                   s   t | d |du rt I dH n|}|rtjntj}|dur'|dvr'tdtj| t|||d}z|j	
|I dH }|durKt||jj|  W dS W dS  tyZ   |sW Y dS w )a/  Create a new Volume object.

        **Examples:**

        ```python notest
        modal.Volume.objects.create("my-volume")
        ```

        Volumes will be created in the active environment, or another one can be specified:

        ```python notest
        modal.Volume.objects.create("my-volume", environment_name="dev")
        ```

        By default, an error will be raised if the Volume already exists, but passing
        `allow_existing=True` will make the creation attempt a no-op in this case.

        ```python notest
        modal.Volume.objects.create("my-volume", allow_existing=True)
        ```

        Note that this method does not return a local instance of the Volume. You can use
        `modal.Volume.from_name` to perform a lookup after creation.

        Added in v1.1.2.

        VolumeN>   r   rI   z&VolumeFS version must be either 1 or 2deployment_namerj   object_creation_typerh   )r4   r7   from_envr   &OBJECT_CREATION_TYPE_CREATE_IF_MISSING*OBJECT_CREATION_TYPE_CREATE_FAIL_IF_EXISTSr   VolumeGetOrCreateRequestr   stubVolumeGetOrCreaterG   metadatarh   r   )rd   rh   ri   rj   rk   ro   reqresponserA   rA   rB   create   s2   
$z_VolumeManager.create )max_objectscreated_beforerj   rk   r{   r|   _Volumec                    s    du rt  I dH n  durdk rtdg dtdtf fdd}|t|I dH }	 |r7n|d	 jjjI dH }q4 fd
dD }durX|d S |S )a  Return a list of hydrated Volume objects.

        **Examples:**

        ```python
        volumes = modal.Volume.objects.list()
        print([v.name for v in volumes])
        ```

        Volumes will be retreived from the active environment, or another one can be specified:

        ```python notest
        dev_volumes = modal.Volume.objects.list(environment_name="dev")
        ```

        By default, all named Volumes are returned, newest to oldest. It's also possible to limit the
        number of results and to filter by creation date:

        ```python
        volumes = modal.Volume.objects.list(max_objects=10, created_before="2025-01-01")
        ```

        Added in v1.1.2.

        Nr   zmax_objects cannot be negativer|   r<   c                    s   d u rdnt dt }tj|| d}tjt|d} j|I d H }|j	 t|j	|k p?d uo?tk}|S )Nd   )r{   r|   )rj   
pagination)
minlenr   ListPaginationVolumeListRequestr   rt   
VolumeListextenditems)r|   max_page_sizer   rw   respfinishedrk   rj   r   r{   rA   rB   retrieve_page   s   "z*_VolumeManager.list.<locals>.retrieve_pageTc                    s.   g | ]}t j|j |jd t |jdqS )Tis_another_apprep)r}   _new_hydrated	volume_idrv   _reprlabel).0itemrk   rj   rA   rB   
<listcomp>  s    z'_VolumeManager.list.<locals>.<listcomp>)	r7   rp   r   floatboolr5   rv   creation_infore   )r{   r|   rj   rk   r   r   volumesrA   r   rB   list   s    !
z_VolumeManager.list)allow_missingrj   rk   r   c                   sb   zt j| |d|I dH }W n ty   |s Y dS w tj|jd}|jj	|I dH  dS )a%  Delete a named Volume.

        Warning: This deletes an *entire Volume*, not just a specific file.
        Deletion is irreversible and will affect any Apps currently using the Volume.

        **Examples:**

        ```python notest
        await modal.Volume.objects.delete("my-volume")
        ```

        Volumes will be deleted from the active environment, or another one can be specified:

        ```python notest
        await modal.Volume.objects.delete("my-volume", environment_name="dev")
        ```

        Added in v1.1.2.

        rj   Nr   )
r}   	from_namehydrater   r   VolumeDeleteRequest	object_id_clientrt   VolumeDelete)rd   r   rj   rk   objrw   rA   rA   rB   delete  s   z_VolumeManager.delete)rM   rN   rO   rP   staticmethodr`   r   rD   r   r7   ry   r   r   builtinsr   r   rA   rA   rA   rB   rg      sb    <Crg   c                   @   sv  e Zd ZU dZdZeej ed< ded< dZ	e
ed< edefd	d
Zedee fddZdZddZdee fddZdee fddZdd Zede
fddZedddddddedee de
dddee dd fdd Zeedddefd!ed  dee dee ddd"eded# fd$d%Z e				d[d&edee dee dddef
d'd(Z!e				d[d&edee dee dddef
d)d*Z"e#de$fd+d,Z%e#d\d.d/Z&e#d0d1 Z'e#d2d3 Z(e)d-d4d5ed6e
de*e+ fd7d8Z,e#dd4d5ed6e
de-e+ fd9d:Z.e)d5edee/df fd;d<Z0e#	d]d5ed=e1j2e/ d>ee3d?e4f  de5fd@dAZ6e#				d[d5ed=e1j2e/ dBee5 dCeej7 dDeej7 d>ee3d?e4f  de5fdEdFZ8e#d^d5ed6e
ddfdGdHZ9e#d^dIe:e dJed6e
ddfdKdLZ;e<ed^dMe
dedN fdOdPZ=e#dQdR Z>ed_dedee dee fdSdTZ?edddUdVedWedee dee fdXdYZ@dS )`r}   a  A writeable volume that can be used to share files between one or more Modal functions.

    The contents of a volume is exposed as a filesystem. You can use it to share data between different functions, or
    to persist durable state across several instances of the same function.

    Unlike a networked filesystem, you need to explicitly reload the volume to see changes made since it was mounted.
    Similarly, you need to explicitly commit any changes you make to the volume for the changes to become visible
    outside the current container.

    Concurrent modification is supported, but concurrent modifications of the same files should be avoided! Last write
    wins in case of concurrent modification of the same file - any data the last writer didn't have when committing
    changes will be lost!

    As a result, volumes are typically not a good fit for use cases where you need to make concurrent modifications to
    the same file (nor is distributed file locking supported).

    Volumes can only be reloaded if there are no open files for the volume - attempting to reload with open files
    will result in an error.

    **Usage**

    ```python
    import modal

    app = modal.App()
    volume = modal.Volume.from_name("my-persisted-volume", create_if_missing=True)

    @app.function(volumes={"/root/foo": volume})
    def f():
        with open("/root/foo/bar.txt", "w") as f:
            f.write("hello")
        volume.commit()  # Persist changes

    @app.function(volumes={"/root/foo": volume})
    def g():
        volume.reload()  # Fetch latest changes
        with open("/root/foo/bar.txt", "r") as f:
            print(f.read())
    ```
    N_lockz'typing.Optional[api_pb2.VolumeMetadata]	_metadataF
_read_onlyr<   c                 C   s   t S N)rg   )r^   rA   rA   rB   objectsh  s   z_Volume.objectsc                 C      | j S r   )_nameselfrA   rA   rB   rd   l  s   z_Volume.namec              	      sD   dt dtdtdtt f fdd}t j|dd fd	d
 jd}|S )a(  Configure Volume to mount as read-only.

        **Example**

        ```python
        import modal

        volume = modal.Volume.from_name("my-volume", create_if_missing=True)

        @app.function(volumes={"/mnt/items": volume.read_only()})
        def f():
            with open("/mnt/items/my-file.txt") as f:
                return f.read()
        ```

        The Volume is mounted as a read-only volume in a function. Any file system write operation into the
        mounted volume will result in an error.

        Added in v1.0.5.
        
new_volumeresolverload_contextexisting_object_idc                    s   |    d| _d S )NT)_initialize_from_otherr   )r   r   r   r   r   rA   rB   _load  s   

z _Volume.read_only.<locals>._loadzVolume()Tc                      s    gS r   rA   rA   r   rA   rB   <lambda>  s    z#_Volume.read_only.<locals>.<lambda>)hydrate_lazilydepsload_context_overrides)r}   r"   r   r   r`   _from_loader_load_context_overrides)r   r   r   rA   r   rB   	read_onlyp  s"   
z_Volume.read_onlyrv   c                 C   s*   |rt |tjs
J || _|j| _d S d S r   )
isinstancer   VolumeMetadatar   rd   r   )r   rv   rA   rA   rB   _hydrate_metadata  s
   z_Volume._hydrate_metadatac                 C   r   r   )r   r   rA   rA   rB   _get_metadata  s   z_Volume._get_metadatac                    s   | j d u rt | _ | j S r   )r   asyncioLockr   rA   rA   rB   	_get_lock  s   


z_Volume._get_lockc                 C   s   | j jd tjjtjjfv S r   )r   rh   r   r>   r?   r@   r   rA   rA   rB   _is_v1  s
   z_Volume._is_v1)	namespacerj   create_if_missingrh   rk   rd   rj   r   rh   z>typing.Optional[modal_proto.api_pb2.VolumeFsVersion.ValueType]rk   c             	      sb   t d t|d dtdtdtdtt f fdd}t|}tj||d	t||d
dS )a  Reference a Volume by name, creating if necessary.

        This is a lazy method that defers hydrating the local
        object with metadata from Modal servers until the first
        time is is actually used.

        ```python
        vol = modal.Volume.from_name("my-volume", create_if_missing=True)

        app = modal.App()

        # Volume refers to the same object, even across instances of `app`.
        @app.function(volumes={"/data": vol})
        def f():
            pass
        ```
        rl   zmodal.Volume.from_namer   r   r   r   c                    sd   t j|j rt jnd d}|jj|I d H }d ur&t|jj	 | 
|j|j|j d S )Nrm   )r   rs   rj   rq   rk   rt   ru   rG   rv   rh   _hydrater   )r   r   r   r   rw   rx   r   rd   rh   rA   rB   r     s   z _Volume.from_name.<locals>._loadTr   )r   rd   r   )	r4   r1   r}   r"   r   r   r`   r   r   )rd   r   rj   r   rh   rk   r   r   rA   r   rB   r     s&   


z_Volume.from_namer^   _heartbeat_sleep)r}   Nc              	     s    du rt  I dH  tjtjt||d jI dH }t 4 I dH ,}tj	|j
d|j fdd|d | j|j
 |jddd	V  W d  I dH  dS 1 I dH sYw   Y  dS )
af  Creates a new ephemeral volume within a context manager:

        Usage:
        ```python
        import modal
        with modal.Volume.ephemeral() as vol:
            assert vol.listdir("/") == []
        ```

        ```python notest
        async with modal.Volume.ephemeral() as vol:
            assert await vol.listdir("/") == []
        ```
        N)ro   rj   rh   r   c                      s    j S r   )rt   VolumeHeartbeatrA   rk   requestrA   rB   r   
  s    z#_Volume.ephemeral.<locals>.<lambda>)sleepTzmodal.Volume.ephemeral()r   )r7   rp   r   rs   OBJECT_CREATION_TYPE_EPHEMERALr   rt   ru   r#   VolumeHeartbeatRequestr   infinite_loopr   rv   )r^   rk   rj   rh   r   rx   tcrA   r   rB   	ephemeral  s(   
.z_Volume.ephemeralrn   c                    s$   t dd t| ||||I dH S )mdmd:hidden)        zThe undocumented `modal.Volume.create_deployed` method is deprecated and will be removed in a future release. It can be replaced with `modal.Volume.objects.create`.N)r0   r}   _create_deployed)rn   r   rk   rj   rh   rA   rA   rB   create_deployed  s   	z_Volume.create_deployedc                    s\   t | d t|d |du rt I dH }tj| t|tj|d}|j	|I dH }|j
S )r   rl   zmodal.Volume.create_deployedNrm   )r4   r1   r7   rp   r   rs   r   rr   rt   ru   r   )rn   r   rk   rj   rh   r   r   rA   rA   rB   r   #  s   
	
z_Volume._create_deployedc                    s:   |   }|s
t S |j}t|jpdt|j|jpddS )z+Return information about the Volume object.N)rd   re   rf   )r   rc   r   rd   r6   re   rf   )r   rv   r   rA   rA   rB   info9  s   z_Volume.infoTc              	      sv   |r
|   I d H nt 4 I d H  tj| jd}| jj|I d H }W d   I d H  d S 1 I d H s4w   Y  d S Nr   )r   r'   r   VolumeReloadRequestr   r   rt   VolumeReload)r   lockrw   _rA   rA   rB   
_do_reloadF  s
   ".z_Volume._do_reloadc                    s   |   I dH 4 I dH E tj| jd}z| jjj|tdddI dH }|js0| j	ddI dH  W n t
tfyE } ztt|d}~ww W d  I dH  dS 1 I dH sWw   Y  dS )zCommit changes to a mounted volume.

        If successful, the changes made are now persisted in durable storage and available to other containers accessing
        the volume.
        Nr   Z   )max_retriesr(   F)r   )r   r   VolumeCommitRequestr   r   rt   VolumeCommitr2   skip_reloadr   r   r   RuntimeErrorr`   )r   rw   r   excrA   rA   rB   commitL  s   .z_Volume.commitc              
      sv   z
|   I dH  W dS  ttfy: } z!t|}d|v r2d| j }t|}|r2t| d| t|d}~ww )a  Make latest committed state of volume available in the running container.

        Any uncommitted changes to the volume, such as new or modified files, may implicitly be committed when
        reloading.

        Reloading will fail if there are open files for the volume.
        Nz-there are open files preventing the operationz/__modal/volumes/z: )r   r   r   r`   r   _open_files_error_annotationr   )r   r   messagevol_path
annotationrA   rA   rB   reload^  s   	z_Volume.reload	recursiverY   r   c                C  s   | dr
td| drtd| jr=tj| j||d}| jjj	|2 z3 dH W }|j
D ]}t|V  q0q'6 dS tj| j||d}| jjj	|2 z3 dH W }|j
D ]}t|V  qWqN6 dS )a  Iterate over all files in a directory in the volume.

        Passing a directory path lists all files in the directory. For a file path, return only that
        file's description. If `recursive` is set to True, list all files and folders under the path
        recursively.
        z**zGlob patterns in `volume get` and `Volume.listdir()` are deprecated. Please pass recursive=True instead. For the CLI, just remove the glob suffix.*zgGlob patterns in `volume get` and `Volume.listdir()` are deprecated. Please remove the glob `*` suffix.r   rY   r   N)endswithr   r   r   VolumeListFilesRequestr   r   rt   VolumeListFilesunary_streamentriesrX   r_   VolumeListFiles2RequestVolumeListFiles2)r   rY   r   rw   batchentryrA   rA   rB   iterdiry  s,   



z_Volume.iterdirc                   s    dd | j ||d2 I dH S )a   List all files under a path prefix in the modal.Volume.

        Passing a directory path lists all files in the directory. For a file path, return only that
        file's description. If `recursive` is set to True, list all files and folders under the path
        recursively.
        c                    s   g | z3 d H W }|q6 S r   rA   )r   r   rA   rA   rB   r     s    z#_Volume.listdir.<locals>.<listcomp>r   N)r   )r   rY   r   rA   rA   rB   listdir  s   z_Volume.listdirc           	   
     s   t j| j|d}z| jj|I dH  W n tjjy* } zt	|j
d d}~ww tdddddtdtfd	d
}dtt f fdd}t }tt| ||d4 I dH }|2 z	3 dH W }|V  q[6 W d  I dH  dS 1 I dH sww   Y  dS )a  
        Read a file from the modal.Volume.

        Note - this function is primarily intended to be used outside of a Modal App.
        For more information on downloading files from a Modal Volume, see
        [the guide](https://modal.com/docs/guide/volumes).

        **Example:**

        ```python notest
        vol = modal.Volume.from_name("my-modal-volume")
        data = b""
        for chunk in vol.read_file("1mb.csv"):
            data += chunk
        print(len(data))  # == 1024 * 1024
        ```
        r   rY   Nr   rL   皙?
n_attempts
base_delaytimeout	block_urlr<   c              	      s`   t  | 4 I d H }|  |j I d H W  d   I d H  S 1 I d H s)w   Y  d S r   )r3   get_sessiongetraise_for_statuscontentread)r  get_responserA   rA   rB   
read_block  s
   0z%_Volume.read_file.<locals>.read_blockc                    s    j D ]} | V  qd S r   )get_urls)urlrx   rA   rB   	iter_urls  s   
z$_Volume.read_file.<locals>.iter_urlsconcurrency)r   VolumeGetFile2Requestr   r   rt   VolumeGetFile2modal	exceptionr   FileNotFoundErrorargsr(   r`   bytesr   multiprocessing	cpu_countr$   r&   )	r   rY   rw   r   r  r  prefetch_num_blocksstreamvaluerA   r  rB   	read_file  s"   .z_Volume.read_filefileobjprogress_cb.c                    s   | j |||dI dH S )zGmdmd:hidden
        Read volume file into file-like IO object.
        )r"  N)_read_file_into_fileobj)r   rY   r!  r"  rA   rA   rB   read_file_into_fileobj  s   
z_Volume.read_file_into_fileobjr  download_semaphorerpc_semaphorec                    sH  d u r	dd |d u rt  }tj| j|d}|d ur|nt }|4 I d H - z| jj|I d H }	W n t	j
jyJ }
 zt|
jd d }
~
ww W d   I d H  n1 I d H s[w   Y  d u rit|t  tddd ddtffd	d
  fddt|	jD }ttj| I d H }|  |S )Nc                  _      d S r   rA   r   __rA   rA   rB   r"    s   z4_Volume._read_file_into_fileobj.<locals>.progress_cbr   r   rL   r  r  r<   c                    sB  | t   }d} 4 I d H  t |4 I d H c}|  |j 2 zM3 d H W }d}|t|k rl4 I d H  || |  	|}W d   I d H  n1 I d H sXw   Y  ||7 }|d |t|k s3|t|7 }q%6 W d   I d H  n1 I d H sw   Y  W d   I d H  |S 1 I d H sw   Y  |S )Nr   )advance)
r*   r3   r  r  r	  r
  iter_anyr   seekwrite)idxr  block_start_posnum_bytes_writtenr  chunknum_chunk_bytes_writtenn)r%  r!  r"  	start_pos
write_lockrA   rB   download_block  s*   &(

:z7_Volume._read_file_into_fileobj.<locals>.download_blockc                    s   g | ]	\}} ||qS rA   rA   )r   r.  r  )r6  rA   rB   r         z3_Volume._read_file_into_fileobj.<locals>.<listcomp>)r  r  r   r  r   r'   r   rt   r  r  r  r   r  r  r   	Semaphorer   tellr(   rD   	enumerater  sumgatherr,  )r   rY   r!  r  r%  r&  r"  rw   rpc_ctxrx   r   coros
total_sizerA   )r6  r%  r!  r"  r4  r5  rB   r#    s4   
(
z_Volume._read_file_into_fileobjc              
      s   | j rtdz/| jr"tj| j||d}| jj|I dH  W dS tj	| j||d}| jj
|I dH  W dS  tjjyL } zt|jd d}~ww )z)Remove a file or directory from a volume.&Read-only Volume can not be written tor   Nr   )r   r   r   r   VolumeRemoveFileRequestr   r   rt   VolumeRemoveFileVolumeRemoveFile2RequestVolumeRemoveFile2r  r  r   r  r  )r   rY   r   rw   r   rA   rA   rB   remove_file  s   z_Volume.remove_file	src_pathsdst_pathc                    s   | j rtd| jr,|rtdtj| j|||d}| jjj	|t
dddI dH  dS tj| j|||d}| jjj|t
dddI dH  dS )a(  
        Copy files within the volume from src_paths to dst_path.
        The semantics of the copy operation follow those of the UNIX cp command.

        The `src_paths` parameter is a list. If you want to copy a single file, you should pass a list with a
        single element.

        `src_paths` and `dst_path` should refer to the desired location *inside* the volume. You do not need to prepend
        the volume mount path.

        **Usage**

        ```python notest
        vol = modal.Volume.from_name("my-modal-volume")

        vol.copy_files(["bar/example.txt"], "bar2")  # Copy files to another directory
        vol.copy_files(["bar/example.txt"], "bar/example2.txt")  # Rename a file by copying
        ```

        Note that if the volume is already mounted on the Modal function, you should use normal filesystem operations
        like `os.rename()` and then `commit()` the volume. The `copy_files()` method is useful when you don't have
        the volume mounted as a filesystem, e.g. when running a script on your local computer.
        r@  z+`recursive` is not supported for V1 volumes)r   rF  rG  r   r   r  r   N)r   r   r   
ValueErrorr   VolumeCopyFilesRequestr   r   rt   VolumeCopyFilesr2   VolumeCopyFiles2RequestVolumeCopyFiles2)r   rF  rG  r   r   rA   rA   rB   
copy_files,  s   
"
"z_Volume.copy_filesforce)#_AbstractVolumeUploadContextManagerNc                 C  s   | j rtdtj| jj| j| j|d}| I dH  z|V  W t	
 \}}}||||I dH  dS t	
 \}}}||||I dH  w )a:  
        Initiate a batched upload to a volume.

        To allow overwriting existing files, set `force` to `True` (you cannot overwrite existing directories with
        uploaded files regardless).

        **Example:**

        ```python notest
        vol = modal.Volume.from_name("my-modal-volume")

        with vol.batch_upload() as batch:
            batch.put_file("local-path.txt", "/remote-path.txt")
            batch.put_directory("/local/directory/", "/remote/directory")
            batch.put_file(io.BytesIO(b"some data"), "/foobar")
        ```
        r@  )rO  N)r   r   rP  resolver   rh   r   r   
__aenter__sysexc_info	__aexit__)r   rO  version_context_managerexc_type	exc_value	tracebackrA   rA   rB   batch_uploadV  s   z_Volume.batch_uploadc                    s$   | j jtj| jdI d H  d S r   )r   rt   r   r   r   r   r   rA   rA   rB   _instance_deletew  s   "z_Volume._instance_deletec                    s(   t dd tjj| ||dI dH  dS )aA  mdmd:hidden
        Delete a named Volume.

        Warning: This deletes an *entire Volume*, not just a specific file.
        Deletion is irreversible and will affect any Apps currently using the Volume.

        DEPRECATED: This method is deprecated; we recommend using `modal.Volume.objects.delete` instead.

        )r   r      z^`modal.Volume.delete` is deprecated; we recommend using `modal.Volume.objects.delete` instead.)rj   rk   N)r0   r}   r   r   )rd   rk   rj   rA   rA   rB   r   {  s   z_Volume.deleter   old_namenew_namec                   sD   t j| |d|I d H }tj|j|d}|jj|I d H  d S )Nr   )r   rd   )	r}   r   r   r   VolumeRenameRequestr   r   rt   VolumeRename)r]  r^  rk   rj   r   rw   rA   rA   rB   rename  s   z_Volume.rename)r<   r}   )NNNNTr   )F)NN)ArM   rN   rO   rP   r   r   r   r   ra   r   r   r   rg   r   propertyr`   rd   r   r   r   r   r   r   r   r7   r   rb   r   r   rZ   r   r   r   r   r   r   rc   r   r   r   r   r!   r   rX   r   r   r   r  r   typingIOr   r   rD   r$  r8  r#  rE  r   rN  r    rZ  r[  r   ra  rA   rA   rA   rB   r}   :  s2  
 )
%4(

""	*C")
"r}   vo)type_prefixc                   @   s   e Zd Zdd Zdd Z	ddeeeee	f dee
ef dee fd	d
Z	ddeeef dee
ef defddZe		ddddedeedef  dedd f
ddZdS )rP  c                       d S r   rA   r   rA   rA   rB   rR        z._AbstractVolumeUploadContextManager.__aenter__c                    rh  r   rA   )r   rW  exc_valexc_tbrA   rA   rB   rU    ri  z-_AbstractVolumeUploadContextManager.__aexit__N
local_fileremote_pathmodec                 C   r'  r   rA   )r   rl  rm  rn  rA   rA   rB   put_file     z,_AbstractVolumeUploadContextManager.put_fileT
local_pathr   c                 C   r'  r   rA   )r   rq  rm  r   rA   rA   rB   put_directory  rp  z1_AbstractVolumeUploadContextManager.put_directoryFrh   z-modal_proto.api_pb2.VolumeFsVersion.ValueTyper   r"  .rO  r<   c                 C   sP   | d t jjt jjfv rt||||dS | t jjkr!t||||dS td|  )N)r"  rO  zunsupported volume version: )r   r>   r?   r@   _VolumeUploadContextManagerVOLUME_FS_VERSION_V2_VolumeUploadContextManager2r   )rh   r   rk   r"  rO  rA   rA   rB   rQ    s   z+_AbstractVolumeUploadContextManager.resolver   rb  NF)rM   rN   rO   rR  rU  r   r	   r`   r   r   r
   r   rD   ro  r   rr  r   r   r   rQ  rA   rA   rA   rB   rP    sB    




rP  c                
   @   s  e Zd ZU dZeed< eed< eed< ede	f ed< e
eeg ef ddf  ed< 		d!d
ededeede	f  defddZdd Zdd Z	d"deeeeef deeef dee fddZ	d#deeef deeef defddZdedejfdd ZdS )$rs  z6Context manager for batch-uploading files to a Volume.
_volume_idr   _force.r"  N_upload_generatorsFr   rk   rO  c                 C   s*   || _ || _g | _|pdd | _|| _dS )r   c                  _   r'  r   rA   r(  rA   rA   rB   r         z6_VolumeUploadContextManager.__init__.<locals>.<lambda>N)rw  r   ry  _progress_cbrx  )r   r   rk   r"  rO  rA   rA   rB   __init__  s
   
z$_VolumeUploadContextManager.__init__c                       | S r   rA   r   rA   rA   rB   rR       z&_VolumeUploadContextManager.__aenter__c           
   
      s  |sfdd dt td f f fdd}g }tt| jdd4 I d H }|2 z3 d H W }|| q*6 W d   I d H  n1 I d H sGw   Y  jdd	 tjj	|j
 d
}zjjj|tdddI d H  W d S  ty }	 ztt|	d }	~	ww d S )Nc                  3        j D ]} | E d H  qd S r   )ry  genr   rA   rB   gen_upload_providers     
zC_VolumeUploadContextManager.__aexit__.<locals>.gen_upload_providersr<   c                    s   t  tj 1  fdd D } tdt|  d j d t 	| D ]}|I d H V  q*W d    d S 1 s>w   Y  d S )Nc                    s   g | ]}  |qS rA   )run_in_executor)r   fexelooprA   rB   r     s    zX_VolumeUploadContextManager.__aexit__.<locals>.gen_file_upload_specs.<locals>.<listcomp>Computing checksums for z files using z workers)
r   get_event_loop
concurrentfuturesThreadPoolExecutorr8   debugr   _max_workersas_completed)futsfut)r  r  rB   gen_file_upload_specs  s   "zD_VolumeUploadContextManager.__aexit__.<locals>.gen_file_upload_specs   r  Tcompleter   files!disallow_overwrite_existing_filesr   rH  r   )r   r+   r$   r%   _upload_fileappendr{  r   VolumePutFilesRequestrw  rx  r   rt   VolumePutFilesr2   r   FileExistsErrorr`   )
r   rW  rj  rk  r  r  r  r   r   r   rA   r  r   rB   rU    s.   
 ($z%_VolumeUploadContextManager.__aexit__rl  rm  rn  c                    H   t  drtd d fdd}| j|  dS )Upload a file from a local file or file-like object.

        Will create any needed parent directories automatically.

        If `local_file` is a file-like object it must remain readable for the lifetime of the batch.
        /remote_path (*) must refer to a file - cannot end with /c                   3   B    t  tst  tr fddV  d S  fddV  d S )Nc                      s   t  tS r   )r/   r
   rA   rl  rn  rm  rA   rB   r     s    zC_VolumeUploadContextManager.put_file.<locals>.gen.<locals>.<lambda>c                      s   t  tpdS Ni  )r.   r
   rA   r  rA   rB   r         r   r`   r	   rA   r  rA   rB   r    s   z1_VolumeUploadContextManager.put_file.<locals>.genN)r
   as_posixr   rI  ry  r  r   rl  rm  rn  r  rA   r  rB   ro    s
   
z$_VolumeUploadContextManager.put_fileTrq  r   c                    L   t  s
J tfdd  fdd}| j|  dS )z
        Upload all files in a local directory.

        Will create any needed parent directories automatically.
        c                          fddS )Nc                      s   t   S r   )r/   rA   relpath_strrm  subpathrA   rB   r   (  s    z^_VolumeUploadContextManager.put_directory.<locals>.create_file_spec_provider.<locals>.<lambda>relative_tor  rq  rm  r  r  rB   create_file_spec_provider&     
zL_VolumeUploadContextManager.put_directory.<locals>.create_file_spec_providerc                  3   :    r dnd} | D ]}| r |V  qd S Nr   rglobglobis_filer  r  )r  rq  r   rA   rB   r  *     
z6_VolumeUploadContextManager.put_directory.<locals>.genN)r	   is_dirr
   ry  r  r   rq  rm  r   r  rA   )r  rq  r   rm  rB   rr       z)_VolumeUploadContextManager.put_directory	file_specr<   c              	      s  |j }| j||jd}tj|jd}| jjj|t	dddI d H }t
 }|js|jrxtd|j d|j d | }t|| jjt| j||j|jd	I d H }W d    n1 s^w   Y  td
|j d|  tj||jd}	n2td|j d| d|j d |jd u rt|jI d H }
n|j}
tj|
|jd}	| j|dd t
 | tk r| jjj|	t	dddI d H }|jrnt
 | tk s|jstd|j dn| j|dd tj||j|jdS )Nrd   r\   )
sha256_hexr   rH  r   zCreating blob file for z (z bytes))r  md5_hexzUploading blob file z as )data_blob_idr  zUploading file z to )datar  T)task_idr  zUploading of z
 timed out)filenamer  rn  )mount_filenamer{  r\   r   MountPutFileRequestr  r   rt   MountPutFiler2   time	monotonicexistsuse_blobr8   r  source_descriptionsourcer-   	functoolspartialr  r
  r   	to_threadread_contentVOLUME_PUT_FILE_CLIENT_TIMEOUTr   	MountFilern  )r   r  remote_filenameprogress_task_idr   rx   
start_timefpblob_idrequest2r
  rA   rA   rB   r  3  sT   

z(_VolumeUploadContextManager._upload_filerv  r   rb  )rM   rN   rO   rP   r`   ra   r7   r   r   r   r   r   r+   r   r|  rR  rU  r   r	   r   r   r
   rD   ro  rr  r   r  r  rA   rA   rA   rB   rs    sH   
 

&




rs  c                   @   s  e Zd ZU dZeed< eed< edef ed< e	ed< e
ed< e
ed< eee  ed	< d
de dfdededeedef  de	de
de
fddZdd Zdd Z	
d&deeeeef deeef dee
 fddZ	d'deeef deeef d e	fd!d"Zd#ee fd$d%Zd
S )(ru  z@Context manager for batch-uploading files to a Volume version 2.rw  r   .r{  rx  _hash_concurrency_put_concurrency_uploader_generatorsNF   r   rk   r"  rO  hash_concurrencyput_concurrencyc                 C   s6   || _ || _g | _|pdd | _|| _|| _|| _dS )r   c                  _   r'  r   rA   r(  rA   rA   rB   r     rz  z7_VolumeUploadContextManager2.__init__.<locals>.<lambda>N)rw  r   r  r{  rx  r  r  )r   r   rk   r"  rO  r  r  rA   rA   rB   r|  r  s   

z%_VolumeUploadContextManager2.__init__c                    r}  r   rA   r   rA   rA   rB   rR    r~  z'_VolumeUploadContextManager2.__aenter__c                    sN   |s%fdd dt t f fdd}| I d H }|I d H  d S d S )Nc                  3   r  r   )r  r  r   rA   rB   r    r  zD_VolumeUploadContextManager2.__aexit__.<locals>.gen_upload_providersr<   c                     s`   t j  fdd D } tdt|  d g }t | D ]
}||I d H  q#|S )Nc                    s   g | ]	}t | qS rA   r   create_task)r   r  hash_semaphorerA   rB   r     r7  zY_VolumeUploadContextManager2.__aexit__.<locals>.gen_file_upload_specs.<locals>.<listcomp>r  z files)r   r8  r  r8   r  r   r  r  )uploads
file_specsr  r  r  rB   r    s   zE_VolumeUploadContextManager2.__aexit__.<locals>.gen_file_upload_specs)r   r,   _put_file_specs)r   rW  rj  rk  r  upload_specsrA   r  rB   rU    s   z&_VolumeUploadContextManager2.__aexit__rl  rm  rn  c                    r  )r  r  r  r  c                   3   r  )Nc                    s   t  t| S r   )r,   	from_pathr
   r  r  rA   rB   r     s    zD_VolumeUploadContextManager2.put_file.<locals>.gen.<locals>.<lambda>c                    s   t  t| p
dS r  )r,   from_fileobjr
   r  r  rA   rB   r     s    r  rA   r  rA   rB   r    s   z2_VolumeUploadContextManager2.put_file.<locals>.genN)r
   r  r   rI  r  r  r  rA   r  rB   ro    s
   

z%_VolumeUploadContextManager2.put_fileTrq  r   c                    r  )r  c                    r  )Nc                    s   t   | S r   )r,   r  r  r  rA   rB   r     s    zQ_VolumeUploadContextManager2.put_directory.<locals>.create_spec.<locals>.<lambda>r  r  r  r  rB   create_spec  r  z?_VolumeUploadContextManager2.put_directory.<locals>.create_specc                  3   r  r  r  r  )r  rq  r   rA   rB   r    r  z7_VolumeUploadContextManager2.put_directory.<locals>.genN)r	   r  r
   r  r  r  rA   )r  rq  r   rm  rB   rr    r  z*_VolumeUploadContextManager2.put_directoryr  c           	         s  i  t dt| d tdD ]e}g }|D ]} fdd|jD }|tjj|j	|j
|j|d qtj| j|| j d}z| jjj|tdd	d
I d H }W n tyc } ztt|d }~ww |jsi nt||j | j| jI d H  qtd| jdd d S )Nz	Ensuring z files are uploaded...rI   c                    s&   g | ]}t jj|j |jd qS ))contents_sha256put_response)r   VolumePutFiles2RequestBlockr  r  )r   blockput_responsesrA   rB   r     s    z@_VolumeUploadContextManager2._put_file_specs.<locals>.<listcomp>)rY   rn  r\   blocksr  r   rH  r   zKDid not succeed at uploading all files despite supplying all missing blocksTr  )r8   r  r   ranger  r  r   r  FilerY   rn  r\   rw  rx  r   rt   VolumePutFiles2r2   r   r  r`   missing_blocks_put_missing_blocksr  r{  r   )	r   r  r   r  r  r  r   rx   r   rA   r  rB   r    s@   
"z,_VolumeUploadContextManager2._put_file_specsr   rb  )rM   rN   rO   rP   r`   ra   r7   r   r   r   rD   r   r   _FileUploader2r  r  r   r|  rR  rU  r   r	   r   r   r
   ro  rr  r,   r  rA   rA   rA   rB   ru  g  sZ   
 


 


ru  r  r  r  r  r"  .c           	         s   t G dd d t|t tdt| d dtttf f fddfdd	|D }t	|D ]}|I d H \}}|||< q<d S )
Nc                   @   s"   e Zd ZU eed< ee ed< dS )z)_put_missing_blocks.<locals>.FileProgressr  pending_blocksN)rM   rN   rO   r`   ra   setrD   rA   rA   rA   rB   FileProgress  s   
 r  z
Uploading z missing blocks...r<   c              	      sj  ddl m} t tjjsJ  j }|j j }|j	vr2|j	|j
d}|t d|j	< |j	 }|j j tj|jd}tddd dd	|d
tf fdd}4 I d H 4 | }|||j|j|j d|d}	||	I d H }
W d    n1 sw   Y  W d   I d H  n1 I d H sw   Y  |j j t|jdkr|dd |j|
fS )Nr   )BytesIOSegmentPayloadr  )r  r   )r     g      ?r  payloadr<   c              
      s   | j dd@ t j j| d4 I d H }|  |j I d H W  d   I d H  W  d    S 1 I d H s:w   Y  W d    d S 1 sJw   Y  d S )NT)subtract_progress)r  )reset_on_errorr3   r  putput_urlr	  r
  r  )r  rx   missing_blockrA   rB   put_missing_block_attempt0  s   "zQ_put_missing_blocks.<locals>.put_missing_block.<locals>.put_missing_block_attempti   )
chunk_sizeprogress_report_cbr   Tr  )_utils.bytes_io_segment_payloadr  r   r   VolumePutFiles2ResponseMissingBlock
file_indexr  block_indexrY   r\   r  r   addr  r  r  r(   r  r  startendremover   r  )r  r  r  r  file_task_idfile_progresstask_progress_cbr  	source_fpr  	resp_data)r  file_progressesr  r"  put_semaphorer
  rB   put_missing_block  s:   


	

(

z._put_missing_blocks.<locals>.put_missing_blockc                    s   g | ]	}t  |qS rA   r  )r   r  )r  rA   rB   r   M  r7  z'_put_missing_blocks.<locals>.<listcomp>)
r   r   r8  dictr8   r  r   tupler  r  )	r  r  r  r  r"  taskstask_resultdigestr   rA   )r  r  r  r"  r  r  rB   r    s   	

4
r  
mount_pathc              
      s   t  dkrd S tddtdtt f fdd}td}tdD ] }|	|rEz||}|r8|W   S W q% t
tfyD   Y q%w q%d S )	NLinuxz
/proc/selfpidr<   c           	   	      s6  t d|  dd}| }|d}ddd |D d}W d    n1 s*w   Y  ttd|  d}| rK| krEd	S d
| dS t	d|  dD ]D}z9ttd|  d| }z |
 }| krxd| dW W   S d| d| dW W   S  ty   Y nw W qT ty   Y qTw d S )N/proc/z/cmdlinerb     c                 S   s   g | ]}|  qS rA   )decode)r   partrA   rA   rB   r   ^  r  zP_open_files_error_annotation.<locals>.find_open_file_for_pid.<locals>.<listcomp>z/cwdzcwd is inside volumezcwd of 'z' is inside volumez/fdz/fd/zpath z is openz is open from '')openr  splitjoinrstripr
   osreadlinkis_relative_tor   r  rI  r  )	r'  r  rawpartscmdlinecwdfdrY   rel_pathr%  self_pidrA   rB   find_open_file_for_pidY  s4   


z<_open_files_error_annotation.<locals>.find_open_file_for_pidz^[1-9][0-9]*$r(  )platformsystemr3  r4  r`   r   recompiler   matchr  PermissionError)r%  r>  pid_redirentr   rA   r<  rB   r   S  s"   



r   )sr   r   concurrent.futuresr  enumr  r  r3  r?  rA  rS  r  rd  collections.abcr   r   r   r   dataclassesr   r   ior   pathlibr	   r
   r   r   r   r   r   r   google.protobuf.messager   synchronicityr   synchronicity.async_wrapr   modal.exceptionr  modal_proto.api_pb2modal_protor   r   r   r   r   r   _load_contextr   _objectr   r   r   r   r    r!   	_resolverr"   _utils.async_utilsr#   r$   r%   r&   r'   r(   r)   _utils.blob_utilsr*   r+   r,   r-   r.   r/   _utils.deprecationr0   r1   _utils.grpc_utilsr2   _utils.http_utilsr3   _utils.name_utilsr4   _utils.time_utilsr5   r6   rk   r7   configr8   r  rD   r`   rG   IntEnumrH   rX   rc   rg   VolumeManagerr}   rl   rP  "AbstractVolumeUploadContextManagerrs  VolumeUploadContextManagerr8  r  ru  VolumeUploadContextManager2r   r   r  r  r   rA   rA   rA   rB   <module>   s    	 $ 	
 +    c'  


M