o
    ;i                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
 d dlmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ e	d
Z		ddeeef dede	dedee dee fddZdS )    N)AsyncIterator)PathPurePosixPath)CallableOptionalUnion)
UsageError)TaskContext)logger)_NetworkFileSystem)	FileEntryFileEntryType_Volume-volumeremote_pathlocal_destination	overwriteconcurrencyprogress_cbc                    s   d u r	dd |d u rt ddt  }tkt r"dn|t|td	
fdd}	
fd	d
  fddtD }tj	| g|R  I d H  dd t
j  d S )Nc                  _   s   d S )N )___r   r   G/home/ubuntu/.local/lib/python3.10/site-packages/modal/cli/_download.pyr       s   z%_volume_download.<locals>.progress_cb         @   c                     s
  t trjdd} n} | 2 z]3 d H W } r(d |fI d H  qtj dd }t|j	
|d} rH| }n}| rhr`| rZt| nt| ntd| d||fI d H  q6 tD ]
}dI d H  qxd S )	NT)	recursive*r   /zOutput path 'z?' already exists. Use --force to overwrite the output directoryNN)
isinstancer   iterdirputr   parentas_posixsplitr   pathrelative_tolstripis_direxistsis_fileosremoveshutilrmtreer   range)iteratorentry
start_pathrel_pathoutput_pathr   )is_piper   num_consumersr   qr   r   r   r   producer-   s2   



z"_volume_download.<locals>.producerc                     s  	   I d H \} }|d u rd S zrK|jtjkrJ|j|jd}tj|d}|j2 z3 d H W }t	j
j| |t|d q/6 |dd n|jtjkr|j|jd}| jjddd tj|d}| d:}ttrj|j| |dI d H }nd	}|j2 z3 d H W }|||7 }|t|d q6 W d    n1 sw   Y  td
| d|   |dd n|jtjkr| jddd W   n  w q)NT)namesize)task_id)advancecomplete)parentsexist_okwb)r(   fileobjdownload_semaphorerpc_semaphorer   r   zWrote z
 bytes to )gettyper   FILEr(   r=   	functoolspartial	read_filesysstdoutbufferwritelenr%   mkdiropenr"   r   _read_file_into_fileobjr
   debug	DIRECTORY	task_done)r7   r4   progress_task_idfile_progress_cbchunkfpb)rF   r8   r   r:   rG   r   r   r   consumerM   sV   

z"_volume_download.<locals>.consumerc                    s   g | ]}  qS r   r   ).0r   )r^   r   r   
<listcomp>y   s    z$_volume_download.<locals>.<listcomp>Tr@   )maxmultiprocessing	cpu_count	PIPE_PATHasyncioQueue	Semaphorer2   r	   gatherrN   rO   flush)r   r   r   r   r   r   r;   	consumersr   )r^   rF   r8   r   r9   r   r   r:   r   rG   r   r   _volume_download   s    

 ,
rk   r!   ) re   rK   rb   r.   r0   rN   collections.abcr   pathlibr   r   typingr   r   r   clickr   modal._utils.async_utilsr	   modal.configr
   modal.network_file_systemr   modal.volumer   r   r   rd   strboolintrk   r   r   r   r   <module>   s<   
