o
    ;iO                  
   @   s0  U d dl Z d dlmZmZmZmZ e jdZer#ddl	m
Z
 e
e d dlZd dlZd dlZd dlZd dlZd dlZd dlmZmZmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlm Z m!Z!m"Z" d dl#m$Z$m%Z% d dl&m'Z'm(Z( d dl)m*Z* d dl+m,Z,m-Z- d dl.m/Z/m0Z0 d dl1m2Z2 ddl3m4Z4 ddl5m6Z6m7Z7m8Z8 erd dl9Z:d dlZ:G dd dZ;deddde<e=df de>de>f
ddZ?de2j@d e(d!eAeeB eejC f fd"d#ZDd$e2jEd e'fd%d&ZFeGd'kre*Hd( e2E ZIe jd)ZJee= eKd*< eJdu reLd+eIMeNeJd,O  e'P ZQzeFeIeQ W n e8y1   e*Rd- Y n eSy?   e*Hd. Y nw eTd  g ZUeVejW eKd/< eX D ]!ZYeZ Z[eYj\durreYj\e[krreYj]sreY^ rreU_eY qReUrd0`d1d2 eUD Zae*bd3eceU d4ea d5 e*Hd6 dS dS )7    N)Serviceimport_class_serviceimport_server_serviceimport_single_function_serviceMODAL_TELEMETRY_SOCKET   )instrument_imports)TYPE_CHECKINGAnyOptionalcast)Message)initialize_clustered_function)UserCodeEventLoop)deserializedeserialize_params)TaskContextaclosingsynchronizer)App_App)Client_Client)logger)ExecutionErrorInputCancellation)
RunningApprunning_app_from_layout)api_pb2)execution_context)ContainerIOManager	IOContextUserExceptionc                   @   s2   e Zd ZdefddZdd Zdd Zdd	 Zd
S )DaemonizedThreadPoolmax_threadsc                 C   s
   || _ d S Nr$   )selfr$    r(   O/home/ubuntu/.local/lib/python3.10/site-packages/modal/_container_entrypoint.py__init__7   s   
zDaemonizedThreadPool.__init__c                 C   s   d| _ t | _t | _| S )Nr   )spawned_workersqueueQueueinputs	threadingEventfinishedr'   r(   r(   r)   	__enter__:   s   

zDaemonizedThreadPool.__enter__c                 C   sN   | j   |d u r| j  d S | jjr%td| jj dt|  d S d S )Nz"Exiting DaemonizedThreadPool with z! active inputs due to exception: )r1   setr.   joinunfinished_tasksr   inforepr)r'   exc_type	exc_value	tracebackr(   r(   r)   __exit__@   s   
zDaemonizedThreadPool.__exit__c                    sL    fdd} j  jk rtj|dd    j d7  _  j||f d S )Nc                     s    j  s@z jjdd\} }W n
 tjy   Y q w z| |  W n ty3   td|  d Y nw  j	   j  rd S d S )Nr   )timeoutzException raised by z  in DaemonizedThreadPool worker!)
r1   is_setr.   getr,   EmptyBaseExceptionr   	exception	task_done)_func_argsr2   r(   r)   worker_threadN   s   

z2DaemonizedThreadPool.submit.<locals>.worker_threadT)targetdaemonr   )r+   r$   r/   Threadstartr.   put)r'   funcargsrF   r(   r2   r)   submitM   s
   zDaemonizedThreadPool.submitN)__name__
__module____qualname__intr*   r3   r<   rN   r(   r(   r(   r)   r#   3   s
    r#   user_code_event_loopcontainer_io_managerz6modal._runtime.container_io_manager.ContainerIOManagerfinalized_functionsz2modal._runtime.user_code_imports.FinalizedFunctionbatch_max_sizebatch_wait_msc           	         s&  dt dd ffdddt dd ffddjrTtjd+	
fdd	d
fdd 	f	dd}
|  W d    d S 1 sMw   Y  d S  D ]5}|dd  |jjrp
| q[dd }t		t	j
|}z| W t		t	j
| q[t		t	j
| w d S )N
io_contextreturnc           	         s  t | j| j| j}t } j| |4 I d H  | jj	rt 
 }|d us)J t  }|d us3J  jdI d H } |||  |4 I d H E d}t|  4 I d H #}|2 z3 d H W } j||I d H  |d7 }qZ6 W d   I d H  n1 I d H sw   Y  W d   I d H  n1 I d H sw   Y   j|| ||I d H  n|  I d H } j| ||I d H  W d   I d H  n1 I d H sw   Y  |  d S Ni   r   r   )r   _set_current_context_ids	input_idsfunction_call_idsattempt_tokenstimehandle_input_exceptionaiofinalized_functionis_generatorcurrent_function_call_idcurrent_attempt_token_queue_creategenerator_output_sender_generator_output_formatr   call_generator_async
_queue_put_send_outputsoutput_items_generator_donecall_function_asyncpush_outputs)	rX   reset_context
started_atrd   re   generator_queue
item_countgenvaluerT   r(   r)   run_input_asynch   sL   
*(
(
z&call_function.<locals>.run_input_asyncc           
   	      s  t   }t| j| j| j} | |j | jjrl| 	 }t
 }|d us'J t }|d us1J  d} |||  | d}|D ]} || |d7 }qEW d    n1 s\w   Y   || || n|  }	 | ||	 W d    n1 sw   Y  |  d S rZ   )r_   r   r[   r\   r]   r^   r`   rb   rc   call_generator_syncrd   re   rf   rg   rh   rj   rk   rl   call_function_syncrn   )
rX   rp   ro   rs   rd   re   rq   rr   rt   valuesru   r(   r)   run_input_sync   s@   


z%call_function.<locals>.run_input_syncr&   c                    s    fdd}|S )Nc                      s   j  j d S r%   )loopcall_soon_threadsafecancelr(   )taskrS   r(   r)   f   s   z<call_function.<locals>.make_async_cancel_callback.<locals>.fr(   )r~   r   )rS   )r~   r)   make_async_cancel_callback   s   z1call_function.<locals>.make_async_cancel_callbackFc                      s,    sd t d tt tj d S d S )NTzUser cancelling input of non-async functions with input concurrency enabled.
This shuts down the container, causing concurrently running inputs to be rescheduled in other containers.)r   warningoskillgetpidsignalSIGINTr(   )
did_sigintr(   r)   cancel_callback_sync   s   z+call_function.<locals>.cancel_callback_syncc               	      s   t d4 I d H :} j 2 z$3 d H W }|jjr+| |}|| q| | q6 W d   I d H  d S 1 I d H sIw   Y  d S )Ng{Gz?)r   run_inputs_outputsra   rb   is_asynccreate_taskset_cancel_callbackrN   )task_contextrX   
input_task)	rV   rW   r   rT   rU   r   rv   rz   thread_poolr(   r)   run_concurrent_inputs   s   .z,call_function.<locals>.run_concurrent_inputsc                   S   s   t t  tjS r%   )r   r   r   r   SIGUSR1r(   r(   r(   r)   <lambda>       zcall_function.<locals>.<lambda>c                 S   s   t d)NzInput was cancelled by user)r   )signum
stackframer(   r(   r)   _cancel_input_signal_handler   s   z3call_function.<locals>._cancel_input_signal_handler)r!   input_concurrency_enabledr#   max_concurrencyrunr   r   rb   r   r   r   )	rS   rT   rU   rV   rW   r   rX   r   usr1_handlerr(   )rV   rW   r   rT   r   rU   r   rv   rz   r   rS   r)   call_functiona   s(   &""/	
 r   function_defclientrY   c                 C   sl   | j tjjkr.| js| jsJ | jrt| j|}nd }| jr(t| j|}||fS d }||fS d\}}||fS )N)NN)definition_typer   FunctionDEFINITION_TYPE_SERIALIZEDfunction_serializedclass_serializedr   )r   r   ser_funser_usr_clsr(   r(   r)   &get_serialized_user_class_and_function   s   r   container_argsc                    s  t | |}| j}|jotjddk}ttt	|}|
  ||_ t A}t||\}}|  | jrDt| j||\}	}
nd}	i }
t ^ |jrl| jj|j   fdd| jjD d }t||||}n7|jr| jj|j   fdd| jjD d }| jj|jd }t||||||	|
}n|d u sJ t||}W d    n1 sw   Y  |j}|jj t!j"j#krd}d}n
|j$pd}|j%pd}W d    n1 sw   Y  t&| j'| j}tt(t)|}|*|| |j+d ur7d	d |j,D }t-|j+t-|krt.d
t-|j+ dt-| d|j+ d| d	t/||j+D ]\}}|j0| }|1||| q%|j2dkrEt3|| j4|j2 |||}t5||||| W d    n	1 s_w   Y  W d    n1 sow   Y  W d    d S W d    d S 1 sw   Y  d S )NMODAL_ENABLE_SNAP_RESTORE1r(   c                       g | ]	}|j  kr|qS r(   	object_id.0oservice_base_function_idr(   r)   
<listcomp>2      zmain.<locals>.<listcomp>r   c                    r   r(   r   r   r   r(   r)   r   @  r   z.*c                 S   s   g | ]}|j qS r(   r   )r   depr(   r(   r)   r   l  r   zFunction has z  dependencies but container got z object ids.
Code deps: z
Object ids: z

This can happen if you are defining Modal objects under a conditional statement that evaluates differently in the local and remote environments.)6r    r   is_checkpointing_functionr   environr?   r   r   r   _translate_inhello
heartbeatsr   r   handle_user_exceptionserialized_paramsr   r   _import_context	is_server
app_layoutfunction_idsfunction_nameobjectsr   is_class	class_idsremovesuffixr   r   apppty_infopty_typer   PTYInfoPTY_TYPE_SHELLrV   batch_linger_msr   app_idr   _translate_out_init_containerservice_depsobject_dependencieslenr   zipobject_handle_metadata_hydrate_experimental_group_sizer   task_idr   )r   r   rT   r   is_snapshotting_function_client
event_loopr   r   
param_argsparam_kwargsservice_function_hydration_dataserviceclass_id
active_apprV   rW   container_appr   dep_object_idsr   objmetadatarU   r(   r   r)   main  s   





&

8	
Tr   __main__zContainer: startingMODAL_CONTAINER_ARGUMENTS_PATHcontainer_arguments_pathz1No path to the container arguments file provided!rbzUser exception caught, exitingzContainer: interruptedlingering_threadsz, c                 c   s    | ]}|j V  qd S r%   )name)r   tr(   r(   r)   	<genexpr>  s    r   z	Detected z background thread(s) [z]] still running after container exit. This will prevent runner shutdown for up to 30 seconds.zContainer: done)dr    modal._runtime.user_code_importsr   r   r   r   r   r?   telemetry_socket_runtime.telemetryr   asyncior,   r   r/   r_   typestypingr	   r
   r   r   google.protobuf.messager   modal._clustered_functionsr   #modal._runtime.user_code_event_loopr   modal._serializationr   r   modal._utils.async_utilsr   r   r   	modal.appr   r   modal.clientr   r   modal.configr   modal.exceptionr   r   modal.running_appr   r   modal_protor   _runtimer   _runtime.container_io_managerr    r!   r"   #modal._runtime.container_io_managermodalr#   dictstrrR   r   r   tupletypeFunctionTyper   ContainerArgumentsr   rO   debugr   r   __annotations__RuntimeErrorParseFromStringopenreadfrom_envr   r7   KeyboardInterruptsleepr   listrI   	enumeratethread	get_identcurrent_threadidentrH   is_aliveappendr5   thread_namesr   r   r(   r(   r(   r)   <module>   s   
.

 

v


*
