o
    ٷi                     @   sh   d dl Z d dlZd dlmZ d dlZd dlmZmZ d dlm	Z	 ddl
mZ e	eZG dd deZdS )	    Nnullcontext)ProfilerActivityprofile)init_logger   )ProfilerBasec                   @   s   e Zd ZU dZdZedB ed< dZeed< e	dedefdd	Z
e	dedB fd
dZe	dd Ze	defddZe	dd ZdS )TorchProfilerz
    Torch-based profiler configured for End-to-End continuous recording.
    Uses 'on_trace_ready' to handle Trace export.
    Compression is offloaded to a background subprocess to avoid blocking the worker loop.
    N	_profiler _trace_templatetrace_path_templatereturnc              	      s   | j durtd|   | j   d| _ |  tj|}|| _| d d tj	tj
 dd td d  fd	d
}ttjtjgtjjdddd|ddddd| _ | j   | d dS )zH
        Start the profiler with the given trace path template.
        Nz*[Rank %s] Stopping existing Torch profiler_rankz.jsonT)exist_ok[Rank z$] Starting End-to-End Torch profilerc              
      s   zL|    td d   ztdd g td d     d W W d S  tyL } ztd d|  W Y d }~W d S d }~ww  tyj } ztd d|  W Y d }~d S d }~ww )	Nr   z] Trace exported to gzipz-fz'] Triggered background compression for z.gzz#] Background gzip failed to start: z] Failed to export trace: )export_chrome_traceloggerinfo
subprocessPopen	Exceptionwarning)pcompress_erre	json_filerank _/home/ubuntu/.local/lib/python3.10/site-packages/vllm_omni/diffusion/profiler/torch_profiler.pytrace_handler4   s   
&$z*TorchProfiler.start.<locals>.trace_handlerr   i )waitwarmupactive)
activitiesscheduleon_trace_readyrecord_shapesprofile_memory
with_stack
with_flops.json.gz)r
   r   r   	_get_rankstopospathabspathr   makedirsdirnamer   r   r   CPUCUDAtorchprofilerr'   start)clsr   r"   r    r   r!   r9      s4   



zTorchProfiler.startc              
   C   s   | j d u rd S |  }| j d| }| d}z| j   W n ty< } ztd| d|  W Y d }~nd }~ww d | _ |d dS )Nr   r-   r   z] Profiler stop failed: )tracetable)r
   r.   r   r/   r   r   r   )r:   r   	base_pathgz_pathr   r    r    r!   r/   \   s   

"
zTorchProfiler.stopc                 C   s   | j d ur| j   d S d S N)r
   stepr:   r    r    r!   r@   s   s   
zTorchProfiler.stepc                 C   s
   | j d uS r?   )r
   rA   r    r    r!   	is_activex   s   
zTorchProfiler.is_activec                 C   s   t  S r?   r   rA   r    r    r!   get_step_context|   s   zTorchProfiler.get_step_context)__name__
__module____qualname____doc__r
   r   __annotations__r   strclassmethodr9   dictr/   r@   boolrB   rC   r    r    r    r!   r	      s   
 @
r	   )r0   r   
contextlibr   r7   torch.profilerr   r   vllm.loggerr   baser   rD   r   r	   r    r    r    r!   <module>   s   