o
    `۷i
C                     @   s   d dl Z d dlZd dlmZ d dlmZmZmZmZm	Z	m
Z
mZ d dlmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZ d d	lmZ edu rTed
e eZe dZ!G dd deZ"G dd dZ#dS )    N)defaultdict)AsyncIterable	AwaitableCallableDictListOptionalTuple)ActorIDNodeIDWorkerID)	BaseModel)ActorTableData)JOB_LOGS_PATH_TEMPLATE)DEFAULT_RPC_TIMEOUTGetLogOptionsprotobuf_to_task_state_dict)StateDataSourceClientz3Please install pydantic via `pip install pydantic`.z0.*worker-([0-9a-f]+)-([0-9a-f]+)-(\d+).(out|err)c                   @   s6   e Zd ZU eed< eed< ee ed< ee ed< dS )ResolvedStreamFileInfonode_idfilenamestart_offset
end_offsetN)__name__
__module____qualname__str__annotations__r   int r   r   [/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/dashboard/modules/log/log_manager.pyr      s
   
 r   c                   @   s  e Zd ZdefddZedefddZdee dee fdd	Z		
d.dede
dedeeee f fddZdedeegeee  f dee fddZdedeeef fddZdedee dee
 dede
dee fddZdedeegeee  f dede
fddZd ed!e
dede
fd"d#Zd$d$d$d$d$d$d$ed%d$d&
dee d'ee dee d ee d!ee
 dee deeegeee  f  de
ded(ee defd)d*Zd+ee deeee f fd,d-Zd$S )/LogsManagerdata_source_clientc                 C   s
   || _ d S Nclient)selfr"   r   r   r    __init__+   s   
zLogsManager.__init__returnc                 C   s   | j S r#   r$   )r&   r   r   r    r"   .   s   zLogsManager.data_source_clientnode_ipc                    s   | j |I dH S )zResolve the node id in hex from a given node ip.

        Args:
            node_ip: The node ip.

        Returns:
            node_id if there's a node id that matches the given node ip and is alive.
            None otherwise.
        N)r%   ip_to_node_id)r&   r)   r   r   r    r*   2   s   
zLogsManager.ip_to_node_id*r   timeoutglob_filterc                    s&   | j j|||dI dH }| |jS )a  Return a list of log files on a given node id filtered by the glob.

        Args:
            node_id: The node id where log files present.
            timeout: The timeout of the API.
            glob_filter: The glob filter to filter out log files.

        Returns:
            Dictionary of {component_name -> list of log files}

        Raises:
            ValueError: If a source is unresponsive.
        r,   N)r%   	list_logs_categorize_log_files	log_files)r&   r   r,   r-   replyr   r   r    r/   >   s   zLogsManager.list_logsoptionsget_actor_fnc                 C  s   |j }|du r| |jI dH }| j||j|j|j|j|j||j	|j
|jd
I dH }|jdk}| jj|j |j||j|j|sA|j	nd|j|jdI dH }|2 z
3 dH W }|jV  qN6 dS )zGenerate a stream of logs in bytes.

        Args:
            options: The option for streaming logs.

        Return:
            Async generator of streamed logs in bytes.
        N
r   log_filenameactor_idtask_idattempt_numberpidr4   r,   suffixsubmission_idstream)r   log_file_name
keep_alivelinesintervalr,   r   r   )r   r*   r)   resolve_filenamer   r7   r8   r9   r:   r,   r;   r<   
media_typer%   
stream_logr@   rA   r   r   data)r&   r3   r4   r   resr?   r=   streamed_logr   r   r    stream_logsQ   s<   

zLogsManager.stream_logs
sub_job_idc                    s   | j jtdI dH }d}|D ]}|j|kr|} nq|du r*td| d dS |j}|du r9td| dtj	|d}||fS )	zReturn the log file name and node id for a given job submission id.

        Args:
            sub_job_id: The job submission id.

        Returns:
            The log file name and node id.
        r.   NzSubmission job ID  not found.)NNzJob zH has no driver node id info. This is likely a bug. Please file an issue.)r<   )
r%   get_job_infor   r<   loggerinfodriver_node_id
ValueErrorr   format)r&   rI   	job_infos
target_jobjob_infor   r6   r   r   r    _resolve_job_filename   s$   	

z!LogsManager._resolve_job_filenamenode_id_hexworker_id_hexr:   r;   c           
         s   |dur|durt d| d| d|dur*| j||d| d| dI dH }n| j||d| d| dI dH }g |d |d D ])}|dur\t|d	}||kr[|  S qEtt|d
}	|	|krn|  S qEdS )zResolve worker log file.NzOnly one of worker id(z	) or pid(z) should beprovided.r+   )r-   
worker_out
worker_err      )rO   r/   WORKER_LOG_PATTERNmatchgroupr   )
r&   rU   rV   r:   r;   r,   r1   r   worker_id_from_filenameworker_pid_from_filenamer   r   r    _resolve_worker_file   s2   	z LogsManager._resolve_worker_filer7   c                    s   |du r	t d||I dH }|du rt d| d|jj}|s*t d| dt|}|jj}|s<t d| dt|}	| j|	 | d||dI dH }
|	 |
fS )	a  Resolve actor log file.

        Args:
            actor_id: The actor id.
            get_actor_fn: The function to get actor information.
            suffix: The suffix of the log file.
            timeout: Timeout in seconds.

        Returns:
            The log file name and node id.

        Raises:
            ValueError: If actor data is not found or get_actor_fn is not provided.
        Nz/get_actor_fn needs to be specified for actor_idz	Actor ID rJ   zWorker ID for Actor ID z' not found. Actor is not scheduled yet.zNode ID for Actor ID rU   rV   r:   r;   r,   )rO   address	worker_idr   r   r   r`   hex)r&   r7   r4   r;   r,   
actor_dataworker_id_binaryrc   node_id_binaryr   r6   r   r   r    _resolve_actor_filename   s4   

z#LogsManager._resolve_actor_filenamer8   r9   c                    s  d}d}d}d}|dvrt d| d| jjdd|fg|dI dH }	t|	jdkr8td	| d
| d| d}
|	jD ]}|j|krH|}
 nq=|
du rXtd| d| dt|
}|dd}|dd}|dd}|dd}|du rtd| d| d|du r|durtd| d| d|du rtd| d| d| d| d| 
|dkrdnd}||d}|du rtd| d| d | |d!| d"d}|d!| d#d}||||fS )$a  Resolve log file for a task.

        Args:
            task_id: The task id.
            attempt_number: The attempt number.
            suffix: The suffix of the log file, e.g. out or err.
            timeout: Timeout in seconds.

        Returns:
            The log file name, node id, the start and end offsets of the
            corresponding task log in the file.

        Raises:
            FileNotFoundError: If the log file is not found.
            ValueError: If the suffix is not out or err.
        NouterrSuffix z is not supported.r8   =)filtersr,   r   z"Could not find log file for task: z
 (attempt z) with suffix: z)Could not find log file for task attempt:()rc   r   task_log_infor7   z)Could not find log file for task attempt.z) due to missing node info.z1For actor task, please query actor log for actor(z): e.g. ray logs actor --id z . Or set RAY_ENABLE_RECORD_ACTOR_TASK_LOGGING=1 in actor's runtime env or when starting the cluster. Recording actor task's log could be expensive, so Ray turns it off by default.z).Worker id = z, node id = z,log_info = rj   stdout_filestderr_filezMissing log filename info in z
 for task z	,attempt std_start_end)	rO   r%   get_all_task_infolenevents_by_taskFileNotFoundErrorr9   r   get)r&   r8   r9   r;   r,   r6   r   r   r   r2   
task_eventttaskrc   log_infor7   filename_keyr   r   r    _resolve_task_filename   s   

z"LogsManager._resolve_task_filenameNrj   r5   r6   r<   c       
            s   d}d}|	dvrt d|	 d|r$| t|||	|I dH \}}n9|r6| |||	|I dH \}}}}n'|
rC| |
I dH \}}n|r]|du rPt d| | j|d||	|dI dH }|du r~td| d| d	| d
| d| d|	 d|
 d| dt||||d}t	
d|  |S )a  Return the file name given all options.

        Args:
            node_id: The node's id from which logs are resolved.
            log_filename: Filename of the log file.
            actor_id: Id of the actor that generates the log file.
            task_id: Id of the task that generates the log file.
            pid: Id of the worker process that generates the log file.
            get_actor_fn: Callback to get the actor's data by id.
            timeout: Timeout for the gRPC to listing logs on the node
                specified by `node_id`.
            suffix: Log suffix if no `log_filename` is provided, when
                resolving by other ids'. Default to "out".
            submission_id: The submission id for a submission job.
        Nri   rl   z is not supported. z=Node id needs to be specified for resolving filenames of pid ra   z^Could not find a log file. Please make sure the given option exists in the cluster.
	node_id: z
	filename: z
	actor_id: z
	task_id: z
	pid: z

	suffix: z
	submission_id: z
	attempt_number: 
)r   r   r   r   zResolved log file: )rO   rh   r
   from_hexr   rT   r`   rz   r   rL   rM   )r&   r   r6   r7   r8   r9   r:   r4   r,   r;   r<   r   r   rF   r   r   r    rB   V  s|   	zLogsManager.resolve_filenamer1   c                 C   s<  t t}|D ]}d|v r|dr|d | qd|v r*|dr*|d | qd|v r;|dr;|d | qd	|v rL|drL|d
 | qd|v rX|d | qd|v rd|d | qd|v rp|d | qd|v r||d | qd|v r|d | qd|v r|d | q|d | q|S )zCategorize the given log files after filterieng them out using a given glob.

        Returns:
            Dictionary of {component_name -> list of log files}
        workerz.outrW   z.errrX   zcore-workerz.logcore_workerzcore-driverdriverzraylet.rayletzgcs_server.
gcs_serverlog_monitorinternalmonitor
autoscalerzagent.agentz
dashboard.	dashboard)r   listendswithappend)r&   r1   resultlog_filer   r   r    r0     s0   z!LogsManager._categorize_log_files)r+   )r   r   r   r   r'   propertyr"   r   r   r*   r   r   r   r/   r   r   r
   r   r   r   bytesrH   r	   rT   r`   rh   r   r   r   rB   r0   r   r   r   r    r!   *   s    

0
'
5
_	
&]r!   )$loggingrecollectionsr   typingr   r   r   r   r   r   r	   rayr
   r   r   ray._common.pydantic_compatr   ray.core.generated.gcs_pb2r    ray.dashboard.modules.job.commonr   ray.util.state.commonr   r   r   ray.util.state.state_managerr   ModuleNotFoundError	getLoggerr   rL   compiler[   r   r!   r   r   r   r    <module>   s     $

