o
    پik                     @   sN  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZmZmZ d dlZd dlmZ d dlmZ eeZe jddd	Zed"defddZe	G dd dZdefddZG dd dZdd Zdd Z dddd
dddej!j"de#e$eee%e f f dB deeej&j'e%gef  dededefd d!Z(dS )#    N)contextmanager)	dataclass)AnyCallableOptionalUnion)CompilationConfig)	rank0_log_COMPILE_ENABLEDF)defaultTenabledc              	   c   s0    t | }zd V  W t | d S t | w N)r
   setreset)r   token r   R/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/compilation/compile.pyset_compiled   s
   
r   c                   @   s   e Zd ZU dZeeejf ed< dZ	e
ee  ed< dZe
ee  ed< dd Zdeeef fd	d
ZdedejfddZdd Zdd ZdefddZdefddZdS )IntermediateTensorsa@  For all pipeline stages except the last, we need to return the hidden
    states and residuals to be sent to the next stage. This data structure
    contains the hidden states and residuals for a request.

    Each stage also needs to handle its own finished_sending and
    finished_recving in case of kv transfer.
    tensorsNfinished_sendingfinished_recvingc                 C   s
   || _ d S r   r   )selfr   r   r   r   __init__-   s   
zIntermediateTensors.__init__keyc                    s@   t  tr
| j  S t  tr|  fdd| j D S d S )Nc                    s   i | ]	\}}||  qS r   r   ).0kvr   r   r   
<dictcomp>8   s    z3IntermediateTensors.__getitem__.<locals>.<dictcomp>)
isinstancestrr   slice	__class__items)r   r   r   r   r   __getitem__4   s
   


zIntermediateTensors.__getitem__valuec                 C   s   || j |< d S r   r   )r   r   r'   r   r   r   __setitem__:      zIntermediateTensors.__setitem__c                 C   s
   | j  S r   )r   r%   r   r   r   r   r%   =      
zIntermediateTensors.itemsc                 C   s
   t | jS r   )lenr   r*   r   r   r   __len__@   r+   zIntermediateTensors.__len__otherc                 C   s   t || jo| S r   )r!   r$   )r   r.   r   r   r   __eq__C   s   zIntermediateTensors.__eq__returnc                 C   s   d| j  dS )NzIntermediateTensors(tensors=)r   r*   r   r   r   __repr__F   r)   zIntermediateTensors.__repr__)__name__
__module____qualname____doc__dictr"   torchTensor__annotations__r   r   r   r   r   r   r#   r&   r(   r%   r-   objectr/   r2   r   r   r   r   r      s   
 r   ndimc                    s*   t | tr| gnt| }  fdd| D S )Nc                    s    g | ]}|d kr
|n | qS )r   r   )r   dr<   r   r   
<listcomp>L   s     z#_normalize_dims.<locals>.<listcomp>)r!   intlist)dimsr<   r   r>   r   _normalize_dimsJ   s   rC   c                   @   s   e Zd ZdZdd ZdS )_MaybeIntermediateTensorszGDuck-typed check to support your IntermediateTensors without importing.c                 C   s&   t |dott|dt| _|| _d S )Nr   )hasattrr!   getattrr7   is_intermediateobj)r   rH   r   r   r   r   R   s   

z"_MaybeIntermediateTensors.__init__N)r3   r4   r5   r6   r   r   r   r   r   rD   O   s    rD   c                 C   sd   t | tjrtj| t|| j d S t| }|jr.|j	j
 D ]}tj|t||j q d S d S r   )r!   r8   r9   _dynamomaybe_mark_dynamicrC   r<   rD   rG   rH   r   values)valrB   mittr   r   r   _mark_dynamic_on_valueY   s   rO   c                 C   s   t | }i }|j D ]F\}}|j}|tju s'tt|dd gd dddkr,d||< qt|dddv sAtdd t|dg D rFd||< q|d	ksN|d
krRd||< q|sYt	d|S )N__args__r   r3    r9   )r   c                 s   s     | ]}t |d ddkV  qdS )r3   rQ   r   N)rF   )r   ar   r   r   	<genexpr>o   s
    
z;_infer_dynamic_arg_dims_from_annotations.<locals>.<genexpr>ztorch.TensorzOptional[torch.Tensor]z;No dynamic dims inferred; pass dynamic_arg_dims explicitly.)
inspect	signature
parametersr%   
annotationr8   r9   rF   any
ValueError)
forward_fnsigdynnamepannr   r   r   (_infer_dynamic_arg_dims_from_annotationsd   s"   




r`   )dynamic_arg_dimsbackend_factorycompile_config	fullgraph
graph_poolmodulera   rb   rc   rd   re   c                   s   t d jjtstdj	|ptd u r+ddlm   fddg dd d
	fd	d
}t	j
j| 
fdd
fdd}t|_S )Ninstall_torch_compiledz)module.__class__.forward must be callabler   )SGLangBackendc                    s    | |S r   r   )gmex)rh   rc   re   r   r   <lambda>   s    z(install_torch_compiled.<locals>.<lambda>F)compiledcompiled_callablec                    s   | urd S t  }|r(|jr(|j}|jjdkr#tj|jjdkr#n|r(|jsz|j	d }W n
 t
y9   Y d S w |j| urAd S |j	durKd S  | d S )N_compilezconvert_frame.pyframer   )sys	_getframef_backf_codeco_nameospathbasenameco_filenamef_locals	Exceptiongetappend)old_codenew_codero   dynamo_frame)compiled_codesrf   original_coder   r   bytecode_hook   s&   


z-install_torch_compiled.<locals>.bytecode_hookc           
         s   d rdS t }|j| g|R i |}|  pi  D ]\}}||jv r8|j| }|dur8t|| q!tjj	
j t| }tj| d}	|	|i | dd< |	d< dS )z$Compile on first use (with flag ON).rl   N)rd   backendTrm   )rT   rU   bindapply_defaultsr%   	argumentsrO   r8   rI   
eval_frameremove_from_cache__code__types
MethodTypecompile)
r   argskwargsr[   bar]   rB   rL   boundrm   )rb   dyn_maprd   stateunbound_fwdr   r   _ensure_compiled   s&   



z0install_torch_compiled.<locals>._ensure_compiledc                    sV   t  }|r d s | g|R i | d }||i |S | g|R i |S )Nrl   rm   )r
   r{   )r   r   r   use_compiledrm   )r   r   r   r   r   
trampoline   s   z*install_torch_compiled.<locals>.trampoline)r	   r$   forwardcallable	TypeErrorr   r`   sglang.srt.compilation.backendrh   r8   rI   convert_frameregister_bytecode_hookr   r   )rf   ra   rb   rc   rd   re   r   r   r   )rh   r   rb   rc   r   r   rd   re   rf   r   r   r   r   rg   |   s"   	
rg   )T))contextvarsrT   loggingru   rp   r   
contextlibr   dataclassesr   typingr   r   r   r   r8   )sglang.srt.compilation.compilation_configr   sglang.srt.utils.commonr	   	getLoggerr3   logger
ContextVarr
   boolr   r   r@   rC   rD   rO   r`   nnModuler7   r"   rA   fxGraphModulerg   r   r   r   r   <module>   sP    
+
