o
    پi"                     @   sl   d Z ddlZddlZddlZddlZddlZddlZddlmZm	Z	m
Z
mZmZ eeZG dd dZdS )zRMerge Chrome trace files from multiple ranks (TP, DP, PP, EP) into a single trace.    N)AnyDictListOptionalTuplec                   @   s&  e Zd ZdZdedefddZdefddZdee fd	d
Zdede	ee
f fddZde	ee
f defddZdede	ee
f de	eef fddZdee	 de	ee
f dee	 fddZde	ee
f de
de
fddZdedee
e
e
e
f fddZdee
 fddZde	eef fddZd S )!ProfileMergerz@Merge profile traces from all parallelism types: TP, DP, PP, EP.
output_dir
profile_idc                 C   sH   || _ || _tj|d| d| _g d| _ddddd| _d	| _d S )
Nmerged-.trace.json.gz)tpdpppepi i@B i'  d   )dp_rankep_rankpp_ranktp_ranki  )	r   r	   ospathjoinmerged_trace_path
rank_typessort_index_multiplierspid_sort_index_threshold)selfr   r	    r   S/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/utils/profile_merger.py__init__   s   

zProfileMerger.__init__returnc           
      C   sV  |   }|std| j tdt| d dg i}g }t|| jdD ]D}| |}td| d|  | 	||}|d 
|d  d|v rU|
|d  |d= | D ]\}}|dkri||vri|||< qYq&|rq||d< t| jd	}	|	t|d
 W d   n1 sw   Y  td| j  tdt|d   | jS )zMerge Chrome traces from all ranks into a single trace.

        Returns:
            Path to merged trace file.

        Raises:
            ValueError: If no trace files found.
        z%No trace files found for profile_id: zFound z trace files to mergetraceEvents)keyzProcessing z with rank info: devicePropertieswbutf-8NzMerged profile saved to: zTotal events merged: )_discover_trace_files
ValueErrorr	   loggerinfolensorted_get_rank_sort_key_extract_rank_info_handle_fileextenditemsgzipopenr   writejsondumpsencode)
r   trace_filesmerged_traceall_device_properties
trace_file	rank_infooutputr"   valuefr   r   r   merge_chrome_traces'   s6   	
z!ProfileMerger.merge_chrome_tracesc                    s^    j  dg}g }|D ]}tj j|}|t| q fdd|D }tt|}|S )zHDiscover trace files matching profile_id (supports TP/DP/PP/EP formats).z*.trace.json.gzc                    s6   g | ]}| d  j ds| dsd|v r|qS )r
   r   z-memory.picklezTP-)endswithr	   .0r>   r   r   r   
<listcomp>]   s    z7ProfileMerger._discover_trace_files.<locals>.<listcomp>)	r	   r   r   r   r   r/   globlistset)r   patternsr7   patternsearch_patternr   rC   r   r&   T   s   
z#ProfileMerger._discover_trace_filesfilenamec                 C   sR   t j|}i }| jD ]}t|  d|}|r&t|d|| d< q|S )z.Extract rank info (TP/DP/PP/EP) from filename.z-(\d+)   _rank)	r   r   basenamer   researchupperintgroup)r   rK   rN   r;   	rank_typematchr   r   r   r-   g   s   
z ProfileMerger._extract_rank_infor;   c                 C   sX   g }| j D ]}| d}||v r||  || d q|r*dd| dS dS )NrM   02d[-]z	[Unknown])r   appendrQ   r   )r   r;   partsrT   rank_keyr   r   r   _create_rank_labels   s   

z ProfileMerger._create_rank_labelr   c              
   C   s   t d|  z5tj|ddd}t|}W d    n1 s!w   Y  dd | D }| |dg ||d< |W S  t	y^ } zt 
d| d	|  dg iW  Y d }~S d }~ww )
NzProcessing file: rtr%   )encodingc                 S   s   i | ]\}}|d kr||qS )r!   r   )rB   r"   r=   r   r   r   
<dictcomp>   s    z.ProfileMerger._handle_file.<locals>.<dictcomp>r!   zFailed to process trace file z: )r(   r)   r1   r2   r4   loadr0   _process_eventsget	Exceptionerror)r   r   r;   r>   tracer<   er   r   r   r.   |   s"   zProfileMerger._handle_fileeventsc                 C   sr   |  |}|D ]/}|ddkr+| |d}|dur+|| jk r+| |||d d< | d|d  |d< q|S )z>Process events: update sort_index and add rank labels to PIDs.nameprocess_sort_indexpidNargs
sort_index )r]   rc   _maybe_cast_intr   _calculate_sort_index)r   rh   r;   
rank_labeleventrk   r   r   r   rb      s   
zProfileMerger._process_eventsrk   c                 C   s0   |}| j  D ]\}}|||d| 7 }q|S )Nr   )r   r0   rc   )r   r;   rk   rm   rT   
multiplierr   r   r   rp      s   z#ProfileMerger._calculate_sort_indexc                    s    |  | t fdddD S )Nc                 3   s"    | ]}  | d dV  qdS )rM   r   N)rc   )rB   rT   r;   r   r   	<genexpr>   s
    
z3ProfileMerger._get_rank_sort_key.<locals>.<genexpr>)r   r   r   r   )r-   tuple)r   r   r   rt   r   r,      s   
z ProfileMerger._get_rank_sort_keyc              	   C   s$   zt |W S  ttfy   Y d S w )N)rR   r'   	TypeError)r   xr   r   r   ro      s
   
zProfileMerger._maybe_cast_intc              
   C   s   t j| jsddiS z?t| jd}t|}W d    n1 s#w   Y  |  }| jt	|
dg t	|dd |D | jt	|
dg dW S  tye } zdd	t| iW  Y d }~S d }~ww )
Nre   zMerged trace file not foundr^   r!   c                 S   s   g | ]}t j|qS r   )r   r   rN   rA   r   r   r   rD      s    z3ProfileMerger.get_merge_summary.<locals>.<listcomp>r#   )merged_filetotal_eventstotal_filessource_filesr	   device_properties_countzFailed to read merged trace: )r   r   existsr   r1   r2   r4   ra   r&   r*   rc   r	   rd   str)r   r>   merged_datar7   rg   r   r   r   get_merge_summary   s$   zProfileMerger.get_merge_summaryN)__name__
__module____qualname____doc__r   r   r?   r   r&   r   rR   r-   r]   r   r.   rb   rp   r   r,   r   ro   r   r   r   r   r   r      s&    -&	

r   )r   rE   r1   r4   loggingr   rO   typingr   r   r   r   r   	getLoggerr   r(   r   r   r   r   r   <module>   s    
