o
    پi                     @  s8  d dl mZ d dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
mZmZmZmZmZ d dlZd dlmZ er<d dlmZ eeejdd	ZerMd dlZd
d Zd3ddZG dd dZeG dd dZeeee
f Zee Z G dd dZ!edd Z"G dd dZ#d4d!d"Z$d5d(d)Z%d6d7d-d.Z&d8d1d2Z'dS )9    )annotationsN)contextmanager)	dataclass)TYPE_CHECKINGAnyCallableDict	GeneratorListSequenceUnion)set_dp_buffer_len)ForwardBatch SGLANG_OPERATIONS_ENABLE_PROFILE0c                 C  s>   t |}td|| d}t|jD ]}|  q|jsJ |jS )Nprimaryinputs)_convert_operations_to_stages_StageExecutorrange
num_stagesnextdoneoutput)r   
operationsstagesexecutor_ r   W/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/batch_overlap/operations.pyexecute_operations   s   

r!   
inputs_arrr   operations_arrdelta_stagesSequence[int]returnc                 C  s   | \}}|\}}|\}}|dksJ |}	t |}
t |}td|
|d}td||d}t|	D ]}|  q.t|j|	 D ]
}|  |  q<t|	D ]}|  qK|jrX|jsZJ |j|jgS )Nr   ar   b)r   r   r   r   r   r   r   )r"   r#   r$   inputs_ainputs_boperations_aoperations_bdelta_stage_adelta_stage_bdelta_stagestages_astages_b
executor_a
executor_br   r   r   r    execute_overlapped_operations   s$   


r4   c                   @  s   e Zd ZdS )YieldOperationN)__name__
__module____qualname__r   r   r   r    r5   =   s    r5   c                   @  s   e Zd ZU ded< ded< dS )ExecutionOperationstr
debug_namer   fnN)r6   r7   r8   __annotations__r   r   r   r    r9   A   s   
 r9   c                   @  sB   e Zd ZdddZd	d
 Zedd Zedd Zedd ZdS )r   r;   r:   r   List[Stage]r   dictc                 C  sP   || _ || _d| _t | _|| _|d }|j| _|j| _	|j
| _|j | _d S )Nr   forward_batch)_debug_name_stages_index
_StateDict_stage_state_stage_outputglobal_dp_buffer_len_global_dp_buffer_lentbo_padded_len_local_dp_buffer_lenglobal_num_tokens_cpu_global_num_tokensdp_padding_mode
is_max_len_is_dp_max_padding)selfr;   r   r   r@   r   r   r    __init__L   s   z_StageExecutor.__init__c              
   C  s   | j rJ | j| j }t| j| j| j| j t| j	 | j d7 |D ],}t|j
d |jdd| ji| jd ur<| jni | _W d    n1 sKw   Y  q$W d    n1 s[w   Y  |  jd7  _d S )Nr;   state   r   )r   rB   rC   r   rH   rJ   rO   rL   _annotate_regionrA   r;   r<   rE   rF   )rP   stageopr   r   r    r   Z   s*   


z_StageExecutor.nextc                 C  s   | j sJ | jS N)r   rF   rP   r   r   r    r   u   s   
z_StageExecutor.outputc                 C  s   | j | jkS rX   )rC   r   rY   r   r   r    r   z   s   z_StageExecutor.donec                 C  s
   t | jS rX   )lenrB   rY   r   r   r    r   ~   s   
z_StageExecutor.num_stagesN)r;   r:   r   r>   r   r?   )	r6   r7   r8   rQ   r   propertyr   r   r   r   r   r   r    r   K   s    


r   c              	   c  s    t r=tjj| ) t|  d V  W d    n1 sw   Y  W d    d S W d    d S 1 s6w   Y  d S d V  d S rX   )_ENABLE_PROFILEtorchautogradprofilerrecord_functionnvtxannotaterR   r   r   r    rU      s   "
rU   c                      sX   e Zd Zdd Z fddZdd Zdd Zd	d
 ZdddZdd Z	dddZ
  ZS )rD   c                 C  s
   i | _ d S rX   _datarY   r   r   r    rQ         
z_StateDict.__init__c                   sB   |dkrt  || d S || jvsJ d| d|| j|< d S )Nrd   `z6` already exist, are you sure you want to override it?)super__setattr__rd   )rP   keyvalue	__class__r   r    rh      s   
z_StateDict.__setattr__c                 C  s
   | j | S rX   rc   rP   itemr   r   r    __getattr__   re   z_StateDict.__getattr__c                 C  s   | j |= d S rX   rc   rm   r   r   r    __delattr__      z_StateDict.__delattr__c                 C     | j |S rX   )rd   poprm   r   r   r    rs      rq   z_StateDict.popvaluesDict[str, Any]c                 C  s"   |  D ]
\}}t| || qd S rX   )itemssetattr)rP   rt   kvr   r   r    update   s   z_StateDict.updatec                 C  rr   rX   )rd   getrm   r   r   r    r{      rq   z_StateDict.getexpect_keysSequence[str]c                 C  sB   t | j t |krtdt| j d|| j  d S )NzUnexpected keys when clearning. This may indicate you do not release memory early enough but leave it to here. list(self._data.keys())=z expect_keys=)setrd   keys	Exceptionlistclear)rP   r|   r   r   r    r      s
   z_StateDict.clear)rt   ru   )r|   r}   )r6   r7   r8   rQ   rh   ro   rp   rs   rz   r{   r   __classcell__r   r   rk   r    rD      s    	
rD   r   List[Operation]r>   c                 C  s4   t | } tt| dd }tdd |D sJ |S )Nc                 S  s
   t | tS rX   )
isinstancer5   )rW   r   r   r    <lambda>   s   
 z/_convert_operations_to_stages.<locals>.<lambda>c                 s  s    | ]	}t |d kV  qdS )r   N)rZ   ).0chunkr   r   r    	<genexpr>   s    z0_convert_operations_to_stages.<locals>.<genexpr>)_decorate_operationsr   _chunk_by_separatorall)r   operation_chunksr   r   r    r      s   r   rv   	List[Any]is_separatorCallable[[Any], bool] Generator[List[Any], None, None]c                 c  sH    g }| D ]}||r|V  g }q| | qt|dkr"|V  d S d S )Nr   )appendrZ   )rv   r   pending_itemsrn   r   r   r    r      s   
r    debug_name_prefixr:   c                   s    fdd| D S )Nc                   s   g | ]}t | qS r   )_decorate_operation)r   rW   r   r   r    
<listcomp>   s    z(_decorate_operations.<locals>.<listcomp>r   )r   r   r   r   r    r      s   r   	operation	Operationc                 C  s.   t | tr| S t|t| dddd | dS )Nr6   unknownop_r   )r;   r<   )r   r5   r9   getattrreplace)r   r   r   r   r    r      s   
r   )r"   r   r#   r   r$   r%   r&   r   )r   r   r&   r>   )rv   r   r   r   r&   r   )r   )r   r   r   r:   )r   r   r   r:   )(
__future__r   os
contextlibr   dataclassesr   typingr   r   r   r   r	   r
   r   r   r]   sglang.srt.layers.dp_attentionr   ,sglang.srt.model_executor.forward_batch_infor   boolintenvironr{   r\   ra   r!   r4   r5   r9   r   Stager   rU   rD   r   r   r   r   r   r   r   r    <module>   s6    (
	8
	
&
	