o
    پi%=                     @  s"  d dl mZ d dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
 d dlZd dlmZ d dlmZmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ erbd dlmZ d dlmZ e Z e rd dl!Z!de!j"j#gde!j"j$j%gde!j"j$j&ggZ'e!(e' e)e*Z+G dd dZ,dS )    )annotationsN)Path)TYPE_CHECKINGListOptional)envs)
ProfileReqProfileReqOutputProfileReqType)ForwardMode)get_global_server_args)is_npu)ProfileMerger)ProfileManager)ScheduleBatch)	Schedulerzprofiler.profilezprofiler.ProfilerActivity.CUDAzprofiler.ProfilerActivity.CPUc                   @  sb   e Zd Zd.ddZ			d/d0ddZ	d1d2d d!Zd3d"d#Z	d1d2d$d%Zd4d(d)Zd5d,d-Z	dS )6SchedulerProfilerMixinselfr   c                 C  s|   t j rt| j| j| jd| _d S d | _d | _	d | _
d | _d | _d | _d | _d | _d | _d | _d| _d| _d| _d | _d S )N)tp_rank	cpu_groupgpu_idF)r   SGLANG_PROFILE_V2getr   r   dp_tp_cpu_groupr   _profile_managertorch_profilertorch_profiler_output_dirprofiler_activities
profile_idprofiler_start_forward_ctprofiler_target_forward_ctprofiler_prefill_ctprofiler_decode_ctprofiler_target_prefill_ctprofiler_target_decode_ctprofile_by_stageprofile_in_progressmerge_profilesrpd_profiler)r    r)   `/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/managers/scheduler_profiler_mixin.pyinit_profiler&   s*   

z$SchedulerProfilerMixin.init_profilerF N
output_dirOptional[str]
start_stepOptional[int]	num_steps
activitiesOptional[List[str]]
with_stackOptional[bool]record_shapesr%   boolr   strr'   profile_prefixprofile_stagesreturnr	   c                 C  s  t j r| jj|||||||||	|
|dS | jrtdddS || _|	| _|d u r/t	
dd}|d u r7ddg}t| | _|| _|| _|| _|| _|
| _|rXt|| jd	 | _|rz| jrjd
| _d
| _|| _|| _n|rs| j| | _n
| j| | _nd | _tdddS )N)r-   r/   r1   r2   r4   r6   r%   r   r'   r9   r:   Fz;Profiling is already in progress. Call /stop_profile first.successmessageSGLANG_TORCH_PROFILER_DIRz/tmpCPUGPU   r   T	Succeeded)r   r   r   r   	configurer&   r	   r%   r'   osgetenvr   
expanduserr   torch_profiler_with_stacktorch_profiler_record_shapesr   r   r9   max
forward_ctr   r!   r"   r#   r$   r    )r   r-   r/   r1   r2   r4   r6   r%   r   r'   r9   r:   r)   r)   r*   init_profileC   sZ   
z#SchedulerProfilerMixin.init_profilestageOptional[ForwardMode]ProfileReqOutput | Nonec              	     s  t j r
| j S |rd|j nd}td| d| j d| j	 d | j
}| j}| j}tjjjtjjjd  fdd	|D }d
|v rddlm} |  tj| jdtt  d| j  d | _| jdkrdd l}ddlm}	 tjdrtd |	 }
| d}|
!| |"  ~tj#$| j% | | _&| j&'d | j&(  | j&)ddd d| _*n,|rtjj+||d ur|nd|d ur|ndt,sd nt-j.t| jd| _/| j/(  d| _*d|v rtj0j1j2dd d| _*d|v r| j3t4 j5krtj06 7  d| _*t8dddS )Nz for r,   zProfiling startsz. Traces will be saved to: z (with profile id: ))r@   rA   c                   s   g | ]
}| v r | qS r)   r)   ).0aactivity_mapr)   r*   
<listcomp>   s    z8SchedulerProfilerMixin.start_profile.<locals>.<listcomp>RPDr   )rpdTracerControlzrpd--TP-.trace.json.gz)RocpdSchema	trace.rpdTzrpd profile rangeF)r2   r4   r6   on_trace_readyMEMi )max_entriesCUDA_PROFILERrC   r<   )9r   r   r   r   manual_startnameloggerinfor   r   r   rH   rI   torchprofilerProfilerActivityr@   CUDArW   
skipCreaterE   pathjoinr8   timer   rpd_profile_pathsqlite3rocpd.schemarZ   existsunlinkconnectwriteSchemacommitdistributedbarrierr   r(   setPythonTracestart	rangePushr&   profile_is_npu	torch_nputensorboard_trace_handlerr   cudamemory_record_memory_historyr   r   base_gpu_idcudartcudaProfilerStartr	   )r   rM   	stage_strr2   r4   r6   torchprof_activitiesrW   rm   rZ   schema
connectionr)   rS   r*   start_profile   sv   









z$SchedulerProfilerMixin.start_profilec              
   C  s.  | j sdS | jdkrdS t| dddkrt| dddkrdS t| dddkr0t| dddkr0dS t| dddkrBt| d	ddkrBdS z3td
 t| j| j}| }|	 }d| d|
dd d|
dd d}td|  W |S  ty } ztjd| dd d|W  Y d }~S d }~ww )Nr,   r   dp_sizerB   dp_rankpp_sizepp_rankmoe_ep_sizemoe_ep_rankzStarting profile merge...z Merged trace: z
 (Events: total_events?z	, Files: total_filesrP   zProfile merge completed: zFailed to merge profiles: T)exc_infoz Merge failed: )r'   r   getattrrb   rc   r   r   r   merge_chrome_tracesget_merge_summaryr   	Exceptionerror)r   mergermerged_pathsummarymerge_messageer)   r)   r*   _merge_profile_traces   s8   
   


z,SchedulerProfilerMixin._merge_profile_tracesc           	      C  s~  t j r
| j S | jstdddS | jjddd | j	r$| j	d }nd}|r.d|j
 nd}td| d	  | jd ur| j  ts| jd
| j g}t| dddkrb|dt| dd  t| dddkrv|dt| dd  t| dddkr|dt| dd  |d| | d }| jtj| j| tj| j | jd ur| j  | j  | j  tj| j | jdkrddlm } |d| j! d | _d | _!| j"d urd| j"v rtj| jt#t$$ d| j d | d }tj%j&'| tj%j&j(d d d| j"v r!| j)t* j+kr!tj%, -  | . }td| j| d | _d| _d | _/tdd | dS )!NFz8Profiling is not in progress. Call /start_profile first.r<   T)parentsexist_ok-r,   zStop profilingz...zTP-r   rB   zDP-r   r   r   zPP-r   r   zEP-r   rY   )rpd_to_chrome_tracer[   r]   rX   z-memoryz.pickle)enabledr_   z)Profiling done. Traces are saved to: %s%sz
Succeeded.)0r   r   r   r   manual_stopr&   r	   r   mkdirr9   ra   rb   rc   r   stoprz   r   r   r   appendrj   export_chrome_tracerE   ri   rd   rt   ru   r   r(   rangePopflushsglang.srt.utils.rpd_utilsr   rl   r   r8   rk   r}   r~   _dump_snapshotr   r   r   r   r   cudaProfilerStopr   r   )	r   rM   stage_prefixstage_suffixfilename_partsfilenamer   memory_profile_pathr   r)   r)   r*   stop_profile   s   









z#SchedulerProfilerMixin.stop_profilebatchr   c                 C  sJ  t j r| jj|jd d S | jr|j r?| jdkr"| 	|j |  jd7  _| j| j
kr;| jr=| jtjd d S d S d S |j rv| jdkrY| jrS| jtjd | 	|j |  jd7  _| j| jkrr| jrt| jtjd d S d S d S |j r}d S td|j | jr| j| jkr|   | jr| j| jkr| 	  d S d S d S )N)forward_moder   rB   )rM   zunsupported profile stage: )r   r   r   r   stepr   r%   
is_prefillr!   r   r#   r&   r   r   EXTEND	is_decoder"   r$   DECODEis_idleRuntimeErrorr    rK   r   )r   r   r)   r)   r*   _profile_batch_predicateQ  sJ   





z/SchedulerProfilerMixin._profile_batch_predicaterecv_reqr   c                 C  s   |j tjkrB|js|jr&| |j|j|j|j|j	|j
|j|j|j|j|jS | |j|j|j|j|j	|j
|j|j|j|j
 |  S |  S N)typer
   START_PROFILEr%   r/   rL   r-   r1   r2   r4   r6   r   r'   r9   r:   r   r   )r   r   r)   r)   r*   ry   y  s:   zSchedulerProfilerMixin.profile)r   r   )Fr,   N)r   r   r-   r.   r/   r0   r1   r0   r2   r3   r4   r5   r6   r5   r%   r7   r   r8   r'   r7   r9   r8   r:   r3   r;   r	   r   )r   r   rM   rN   r;   rO   )r   r   r;   r8   )r   r   r   r   )r   r   r   r   )
__name__
__module____qualname__r+   rL   r   r   r   r   ry   r)   r)   r)   r*   r   %   s    
'H
O!
X(r   )-
__future__r   loggingrE   rk   pathlibr   typingr   r   r   rd   sglang.srt.environr   sglang.srt.managers.io_structr   r	   r
   ,sglang.srt.model_executor.forward_batch_infor   sglang.srt.server_argsr   sglang.srt.utilsr   sglang.srt.utils.profile_mergerr   sglang.srt.utils.profile_utilsr   "sglang.srt.managers.schedule_batchr   sglang.srt.managers.schedulerr   rz   r{   re   ry   rf   NPUr@   patches_apply_patches	getLoggerr   rb   r   r)   r)   r)   r*   <module>   s6    


