o
    پit                     @   sr  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
 d dlmZ d dlmZmZmZmZ d dlmZ d dlmZmZ d dlmZ d d	lmZmZmZmZmZmZm Z  d dl!Z!d dl"m#Z$ ed
dG dd deZ%ed
dG dd de%Z&G dd dZ'de(fddZ)dPde(de*fddZ+dPde*fddZ,deddfddZ-d d! Z.d"d# Z/d$d% Z0d&d' Z1d(e2de(fd)d*Z3d+d, Z4d-d. Z5d/d0 Z6d1d2 Z7d3e(d4e8d5e*fd6d7Z9d3e(fd8d9Z:	dPd:e*de*ded; fd<d=Z;G d>d? d?Z<G d@dA dAZ=G dBdC dCe=Z>G dDd; d;e=Z?dQdEe(dFe*de*fdGdHZ@dee( fdIdJZAe'e&B dKZCdLdM ZDdNdO ZEdS )R    N)ABCabstractmethod)contextmanager)deepcopy)asdict	dataclassfieldsreplace)cached_property)BaseHTTPRequestHandler
HTTPServer)Path)AnyListLiteralOptionalUnionget_argsget_type_hintsT)frozenc                   @   s   e Zd ZdddZdddZeedefddZed	edefd
dZ	edddZ
dddZedefddZedefddZedd ZdS )_FrozenConfigreturnNc                 C   s   |    d S N)_verify_typesself r   Q/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/debug_utils/dumper.py__post_init__   s   z_FrozenConfig.__post_init__c              
   C   s   t t| }t| j}t| D ].}t| |j}|d u rq| ||j }t||s=t| d|j d|j dt|j qd S )N.z: expected z, got )	r   type__name__r   getattrname_unwrap_type
isinstance	TypeError)r   hintscls_namefvalueexpectedr   r   r   r      s   

z_FrozenConfig._verify_typesc                 C   s   d S r   r   clsr   r   r   _env_prefix)      z_FrozenConfig._env_prefix
field_namec                 C   s   |    |  S r   )r.   upper)r-   r0   r   r   r   	_env_name-   s   z_FrozenConfig._env_namec                    s     di  fddt  D S )Nc                    s&   i | ]}|j   |j |jqS r   )r#   _parse_env_fieldr2   default).0r)   r,   r   r   
<dictcomp>4   s    z*_FrozenConfig.from_env.<locals>.<dictcomp>r   )r   r,   r   r,   r   from_env1   s
   
z_FrozenConfig.from_envc                    s6   t |   fdd| D }|rt| fi |S | S )Nc                    s*   i | ]\}}t  |d u r||qS r   )osgetenvr2   )r5   keyr*   r,   r   r   r6   <   s
    z/_FrozenConfig.with_defaults.<locals>.<dictcomp>)r    itemsr	   )r   kwargsactualr   r,   r   with_defaults:   s
   
z_FrozenConfig.with_defaultsc                 C   s"   t | }|rtdd |D S | S )Nc                 s   s     | ]}|t d ur|V  qd S r   )r    )r5   ar   r   r   	<genexpr>G   s    z-_FrozenConfig._unwrap_type.<locals>.<genexpr>)r   next)hintargsr   r   r   r$   C   s   z_FrozenConfig._unwrap_typeenv_namec                 C   s   t t| |S r   )r   _parse_env_valuer8   r9   )rD   r4   r   r   r   r3   J   s   z_FrozenConfig._parse_env_fieldc                 C   s@   | d u s|   s
|S t|tr|  dv S t|trt| S | S )N)true1)stripr%   boollowerint)rawr4   r   r   r   rE   N   s   

z_FrozenConfig._parse_env_valuer   N)r   r   )r!   
__module____qualname__r   r   classmethodr   strr.   r2   r7   r>   staticmethodr    r$   r3   rE   r   r   r   r   r      s"    


	r   c                   @   s   e Zd ZU dZeed< dZee ed< dZ	eed< dZ
eed< dZeed	< dZeed
< dZeed< dZeed< dZeed< dZee ed< dZeed< dZeed< dZeed< dZeed< edefddZedeeeed f  fddZdS )_DumperConfigFenableNfilterz/tmpdirTenable_output_fileenable_output_consoleenable_valueenable_gradenable_model_valueenable_model_gradpartial_nameenable_http_servercleanup_previous<   collective_timeoutz-1server_portr   c                 C   s   dS )NSGLANG_DUMPER_r   r,   r   r   r   r.   j   r/   z_DumperConfig._env_prefixreusec                 C   s*   | j }|dkr	dS t|}|dkrd S |S )Nrd   r   )rb   rK   )r   rL   portr   r   r   server_port_parsedn   s   z _DumperConfig.server_port_parsed)r!   rN   rO   rT   rI   __annotations__rU   r   rQ   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   ra   rK   rb   rP   r.   propertyr   r   rf   r   r   r   r   rS   Y   s&   
 $rS   c                   @   s  e Zd ZdZdefddZdd Zdd Zd	d
 Zdd Z	d>ddZ
edd ZdefddZdedeeef dee fddZdedeeef defddZd>ddZd>ddZd?d efd!d"Zd?d#ed eddfd$d%Z	&	d@d'd(d)ed eddfd*d+Zd#ed,ed ed-ed.ed/ed0ed1eddfd2d3Zd#ed,ed eddfd4d5Zdd6d7ed8ed ed9ee ddf
d:d;Zedefd<d=Z dS )A_Dumpera7  Utility to dump tensors, which can be useful when comparison checking models.

    Example usage:
    dumper.on_forward_pass_start()
    dumper.dump("layer_start__hidden_states", hidden_states, layer_id=self.layer_id)

    Import from non-SGLang system:
    ```
    import sys
    sys.path.append("/YOUR_PATH/sglang/python/sglang/srt/debug_utils")
    from dumper import dumper
    ```

    Then run the program:
    `SGLANG_DUMPER_ENABLE=1 python ...`

    Auto-cleanup old dumps before first write:
    `SGLANG_DUMPER_CLEANUP_PREVIOUS=1 python ...`

    Alternatively, disable at startup and configure via HTTP:
    1. `python ...`
    2. sglang mode:  `curl -X POST http://localhost:30000/dumper/configure -d '{"enable": true}'`
       standalone:   `curl -X POST http://localhost:40000/dumper/configure -d '{"enable": true}'`
    3. `curl -X POST http://localhost:30000/dumper/configure -d '{"enable": true, "filter": "layer_id=[0-3]"}'`
    4. `curl -X POST http://localhost:30000/dumper/reset`

    Related: `sglang.srt.debug_utils.dump_comparator` for dump comparison
    configc                C   s@   || _ |j | _|j | _d| _d| _i | _d | _t	| | _
d S Nr   )_configr^   _http_server_handledr_   _cleanup_previous_handled_dump_index_forward_pass_id_global_ctx_captured_output_data_LocalOnlyBroadcast_rpc_broadcast)r   rj   r   r   r   __init__   s   

z_Dumper.__init__c                 C   sH   |    | jjs
dS |   |  jd7  _tdt  d| j  dS )z#This should be called on all ranks.N   z
[Dumper] [z] on_forward_pass_start id=)_ensure_http_serverrl   rT   _ensure_partial_namerp   printtimer   r   r   r   on_forward_pass_start   s   z_Dumper.on_forward_pass_startc                 C   s   | j rd S d| _ | jj}|d u rd S t| tdd| jjd}t dkrF|d us)J || _|dkr6td d S t	d| |d	 td
|  d S d S )NTSGLANG_DUMPER_ZMQ_BASE_PORTiA  )	base_porttimeout_secondsr   rd   z@[Dumper] Standalone HTTP server disabled, reusing existing portsz/dumper/)prefixtarget	http_portz%[Dumper] HTTP server started on port )
rm   rl   rf   _create_zmq_rpc_broadcastget_int_env_varra   	_get_rankrt   ry   _start_http_server)r   r   rpc_broadcastr   r   r   rw      s*   
z_Dumper._ensure_http_serverc                 C   s<   | j jd u rt| j jd}| j|d td|  d S d S )N)r~   )r]   z[Dumper] Choose partial_name=)rl   r]   _get_partial_namera   	configurery   )r   r#   r   r   r   rx      s
   z_Dumper._ensure_partial_namec                 K   s   dd | j |B  D | _ dS )z
        Example:

        dumper.configure_default(filter='layer_id=[0-3]')
        dumper.set_ctx(layer_id=self.layer_id)
        ...
        dumper.set_ctx(layer_id=None)
        c                 S   s   i | ]\}}|d ur||qS r   r   r5   kvr   r   r   r6      s    z#_Dumper.set_ctx.<locals>.<dictcomp>N)rq   r;   r   r<   r   r   r   set_ctx   s   	z_Dumper.set_ctxr   Nc                 C   s   d| _ d| _i | _d S rk   )ro   rp   rq   r   r   r   r   reset   s   
z_Dumper.resetc                 c   s4    | j d u sJ i | _ z
| j V  W d | _ d S d | _ w r   )rr   r   r   r   r   capture_output   s   
z_Dumper.capture_outputc                 C   s   t | j| j| jdS )N)rj   
dump_indexforward_pass_id)r   rl   ro   rp   r   r   r   r   	get_state   s   z_Dumper.get_statemethodbodyc                C   s   | j j||dS )Nr   r   )rt   "_handle_http_control_request_innerr   r   r   r   r   r   _handle_http_control_request   s   z$_Dumper._handle_http_control_requestc                C   sN   |dkr|   S |dkr| jdi | i S |dkr |   i S td|)Nr   r   r   zUnknown dumper control method: r   )r   r   r   
ValueErrorr   r   r   r   r     s   z*_Dumper._handle_http_control_request_innerc                 K   s   t | jfi || _d S r   )r	   rl   r   r   r   r   r        z_Dumper.configurec                 K   s   | j jdi || _ d S )Nr   )rl   r>   r   r   r   r   configure_default  r   z_Dumper.configure_defaultTsavec                 K   s@   t |}| D ]\}}| j| d| |fd|i| qd S )N_r   )_obj_to_dictr;   dump)r   name_prefixdatar   r<   r#   r*   r   r   r   	dump_dict  s   $z_Dumper.dump_dictr#   c                 K   s(   | j ||||| jjd| jjddd	 d S )NFzDumper.ValueDumper.Grad	r#   r*   extra_kwargsr   rY   enable_curr_gradenable_future_grad	value_taggrad_tag)_dump_innerrl   rY   rZ   )r   r#   r*   r   r<   r   r   r   r     s   
z_Dumper.dumpparammodelztorch.nn.Moduler   c                 K   sD   |  D ]\}}| j| d| |||| jj| jjdddd	 qd S )N__FzDumper.ParamValuezDumper.ParamGradr   )named_parametersr   rl   r[   r\   )r   r   r   r   r<   
param_namer   r   r   r   
dump_model'  s   z_Dumper.dump_modelr   rY   r   r   r   r   c       	         C   s   |    | jjs
d S tdd|i|| j}
| jj }d ur*t|t|
d u r*d S |s2|s2|s2d S | j	dk r;t
d t|}|rJ| j||
||d |rkt|tjrk|j }d urk| j|	i |
dd| i||d |rx| j||||d d S d S )Nr#   rv   z$Dump without on_forward_pass_start())tagtagsr*   r   grad__)r#   tensorr   r   r   )rw   rl   rT   dictrq   rU   research_format_tagsrp   ry   _materialize_value_dump_singler%   torchTensorgrad_register_dump_grad_hook)r   r#   r*   r   r   rY   r   r   r   r   r   r)   gr   r   r   r   ;  sR   


z_Dumper._dump_innerc                   sh   t |tjsd S |jsd S j tddd| it|dtjdd f fdd}|| d S )Nr#   r   r   r   c                    s   j d|  d d S )Nr   )r   r   r*   r   r   )r   )r   captured_forward_pass_idcaptured_tagsr   r   r   r   	grad_hook  s   
z3_Dumper._register_dump_grad_hook.<locals>.grad_hookr   )r%   r   r   requires_gradrp   r   r   register_hook)r   r#   r   r   r   r   r   r   r   r   w  s   	z _Dumper._register_dump_grad_hook)r   r   r   r   c                C   s  |    |  jd7  _t }td|d ur|n| j|| jd|}t|d }t| jjd| jj	  | }	| jj
rtd| d| dt  d|	 d	t| d
t|tjrX|jnd  dt|tjrd|jnd  dt|tjrp|jnd  dt| dt|  | jd u}
|r| jjs|
rt|tjjr|jn|tdi || jd}|
rt|d |d< || j|d < d S | jsd| _tt| jj |	jj ddd t!|t"|	 d S d S d S )Nrv   )r   rankr   z.ptsglang_dump_[z] [z, z] z type= shape= dtype= device=z id=z sample_value=)r*   metar*   r#   T)parentsexist_okr   )#rx   ro   r   r   rp   r   r   rl   rV   r]   rX   ry   rz   r    r%   r   r   shapedtypedeviceidget_truncated_valuerr   rW   nn	Parameterr   _static_meta_deepcopy_or_clonern   _cleanup_old_dumpsparentmkdir_torch_saverQ   )r   r   r   r*   r   r   r   full_kwargsfull_filenamepath	capturingoutput_datar   r   r   r     sb   	



z_Dumper._dump_singlec                 C   s   t  S r   )_compute_static_metar   r   r   r   r     s   z_Dumper._static_metarM   )T)r   T)!r!   rN   rO   __doc__rS   ru   r{   rw   rx   r   r   r   r   r   r   rQ   r   listr   r   r   r   rI   r   r   r   r   r   r   rK   r   r
   r   r   r   r   r   ri   |   s    








	

<
"
;ri   r   c              
   C   s   z8zt | |W W S  ty8 } z#dt|v r3t| t jjr3td| d t| j	|W  Y d }~W S  d }~ww  t
yS } ztd| d W Y d }~d S d }~ww )Nznot pickleablez[Dumper] Observe error=z and try pickling value.dataz" when saving data, skip the tensor)r   r   RuntimeErrorrQ   r%   r   r   ry   r   r   	Exception)r*   r   er   r   r   r     s   r   r`   operation_namer~   c                    sJ   t    fdd}t j|dd}|  z|  W    S    w )Nc                      s.    j dstd d ddd d S d S )N)timeoutz
[Dumper] WARNING: 'z' has not completed after zTs. This usually means not all ranks are participating in this collective operation.
T)flush)waitry   r   	completedr   r~   r   r   watchdog  s   
z*_collective_with_timeout.<locals>.watchdogTr   daemon)	threadingEventThreadstartset)fnr   r~   r   threadr   r   r   _collective_with_timeout  s   	r   c                    sF   t  }|dkrtt nd g t rt fddd| d  d S )Nr   c                      s   t j ddS )Ncuda)r   )distbroadcast_object_listr   object_listr   r   <lambda>  s    z#_get_partial_name.<locals>.<lambda>z*broadcast_object_list in _get_partial_namer   r~   )r   rQ   rz   r   is_initializedr   )r~   r   r   r   r   r     s   
r   base_dirr   c                 C   sZ   dd l }t dkr!| dD ]}| r || td|  qt r+t  d S d S )Nr   zsglang_dump_*z[Dumper] Cleaned up )	shutilr   globis_dirrmtreery   r   r   barrier)r   r  entryr   r   r   r     s   

r   c                   C      t  rt  S dS rk   )r   r   get_rankr   r   r   r   r        r   c                   C   r  )Nrv   )r   r   get_world_sizer   r   r   r   _get_world_size  r	  r  c              	   C   sj   t | tr| S i }t| D ]%}|dr|drqzt| |}t|s(|||< W q ty2   Y qw |S )Nr   )r%   r   rV   
startswithendswithr"   callabler   )objretr   r   r   r   r   r     s   

r   c                 C   s   t | r|  } | S r   )r  )r*   r   r   r   r   /  s   r   r<   c                 C   s   d dd |  D S )N___c                 s   s"    | ]\}}| d | V  qdS )=Nr   r   r   r   r   r@   6  s     z_format_tags.<locals>.<genexpr>)joinr;   )r<   r   r   r   r   5  r   r   c                 C   s   t | tjr
|  S t| S r   )r%   r   r   cloner   )xr   r   r   r   9  s   r   c                  C   s6   t  t d} t  }r|| d< t  }r|| d< | S )N)
world_rank
world_sizesglang_parallel_infomegatron_parallel_info)r   r  _collect_sglang_parallel_info_collect_megatron_parallel_info)resultr  r   r   r   r   B  s   

r   c               
   C   s6  i } z>ddl m}m}m}m}m}m}m}m} | | d< | | d< | | d< | | d< | | d< | | d< | | d	< | | d
< W n t	t
tfyP   d| d< Y nw z8ddlm}	m}
m}m}m}m}m} | | d< | | d< | | d< |	 | d< |
 | d< | | d< | | d< W | S  t	t
tfy   d| d< Y | S w )Nr   )get_moe_expert_parallel_rank"get_moe_expert_parallel_world_sizeget_moe_tensor_parallel_rank"get_moe_tensor_parallel_world_size get_pipeline_model_parallel_rank&get_pipeline_model_parallel_world_sizeget_tensor_model_parallel_rank$get_tensor_model_parallel_world_sizetp_ranktp_sizepp_rankpp_sizemoe_ep_rankmoe_ep_sizemoe_tp_rankmoe_tp_sizeTdistributed_error)get_attention_dp_rankget_attention_dp_sizeget_attention_tp_rankget_attention_tp_sizeget_local_attention_dp_rankget_local_attention_dp_sizeis_dp_attention_enabledenable_dp_attentionattn_tp_rankattn_tp_sizeattn_dp_rankattn_dp_sizelocal_attn_dp_ranklocal_attn_dp_sizedp_attention_error)sglang.srt.distributedr  r  r  r   r!  r"  r#  r$  ImportErrorAttributeErrorAssertionErrorsglang.srt.layers.dp_attentionr.  r/  r0  r1  r2  r3  r4  )infor  r  r  r   r!  r"  r#  r$  r.  r/  r0  r1  r2  r3  r4  r   r   r   r  P  s8   (






$







r  c               
   C   sN  i } zddl m} | | d< | | d< | | d< | | d< | | d< | | d< | | d	< |	 | d
< |
 | d< | | d< | | d< | | d< | | d< | | d< | | d< | | d< | | d< | | d< | | d< | | d< | | d< | | d< | | d< W | S  tttfy   d| d< Y | S w )Nr   )parallel_stater%  r&  r'  r(  dp_rankdp_sizecp_rankcp_sizevpp_rankvpp_sizeep_rankep_sizeetp_ranketp_sizeedp_rankedp_sizetcp_ranktcp_size	etmp_rank	etmp_sizetp_src_rankmp_src_rankdp_src_rankTmegatron_error)megatron.corerC  r#  r$  r!  r"  get_data_parallel_rankget_data_parallel_world_sizeget_context_parallel_rankget_context_parallel_world_size(get_virtual_pipeline_model_parallel_rank.get_virtual_pipeline_model_parallel_world_sizeget_expert_model_parallel_rank$get_expert_model_parallel_world_sizeget_expert_tensor_parallel_rank%get_expert_tensor_parallel_world_sizeget_expert_data_parallel_rank#get_expert_data_parallel_world_size$get_tensor_and_context_parallel_rank*get_tensor_and_context_parallel_world_size)get_expert_tensor_and_model_parallel_rank/get_expert_tensor_and_model_parallel_world_size"get_tensor_model_parallel_src_rankget_model_parallel_src_rankget_data_parallel_src_rankr>  r?  r@  )rB  mpur   r   r   r    s>   
r  r   r   r   c                 C   s6   t | |d}td|f|}tj|jdd}|  d S )Nr   r   0.0.0.0Tr   )_make_http_handlerr   r   r   serve_foreverr   )r   r   r   handler_classserverr   r   r   r   r     s   r   c                    s   G  fdddt }|S )Nc                       s(   e Zd Z fddZdefddZdS )z(_make_http_handler.<locals>._HTTPHandlerc              
      s   | j  s| d d S | j t d  }zC|  }tdt  d| j  d| j||d}t	|
 }| d | dd | d	tt| |   | j| W d S  tyt } z| d
t| W Y d }~d S d }~ww )Ni  z[Dumper#z] HTTP z
 req_body=r      zContent-Typezapplication/jsonContent-Lengthi  )r   r  
send_errorlen_get_request_bodyry   r   r   jsondumpsencodesend_responsesend_headerrQ   end_headerswfilewriter   )r   r   req_bodyr  	resp_bodyr   rm  r   r   do_POST  s(   

z0_make_http_handler.<locals>._HTTPHandler.do_POSTr   c                 S   s0   t | jdd}|dkri S t| j|S )Nrt  r   )rK   headersgetrx  loadsrfileread)r   content_lengthr   r   r   rw    s   z:_make_http_handler.<locals>._HTTPHandler._get_request_bodyN)r!   rN   rO   r  r   rw  r   rm  r   r   _HTTPHandler  s    r  )r   )r   r   r  r   rm  r   ro    s   ro  r}   _ZmqRpcBroadcastc                    s6  ddl }t }t rt nd}|| }dt  d| | }||j	d|  fdd}t
j|d	d
}	|	  td| d  t rbdg|  t fddd|d ng td| d   |dkrg }
t D ]\}}||j}|| |
t|d| d qyt|
S dS )zSA general-purpose minimal RPC to support broadcasting executions to multi processesr   Nrv   ztcp://:ztcp://*:c               
      s   	 z  } t | d | d i | d }|d d}W n  ty< } ztd|  d t|d}W Y d }~nd }~ww | q)NTr   rC   r<   )r  errorz&[Dumper.ZmqRpc] error inside handler: )
recv_pyobjr"   r   ry   rQ   
send_pyobj)reqr  respr   )handlersockr   r   
serve_loop  s    
z-_create_zmq_rpc_broadcast.<locals>.serve_loopTr   z[Dumper.ZmqRpc] rank=z server started at c                      s   t  S r   )r   all_gather_objectr   )all_addresses
local_addrr   r   r     s    z+_create_zmq_rpc_broadcast.<locals>.<lambda>z.all_gather_object in _create_zmq_rpc_broadcastr   z all_addresses=zrank-)
debug_name)zmqr   r   r   r
  _get_local_ip_by_remoteContextsocketREPbindr   r   r   ry   r   	enumerateREQconnectappend_ZmqRpcHandler  )r  r}   r~   r  r   r  re   ctxr  r   handlesiaddr
req_socketr   )r  r  r  r  r   r     s:   

r   c                   @   s,   e Zd ZdZdefddZdefddZdS )	r  z4Proxy object to call remote handler methods via ZMQ.r  c                 C   s   || _ || _d S r   )_socket_debug_name)r   r  r  r   r   r   ru     s   
z_ZmqRpcHandle.__init__method_namec                        fdd}|S )Nc                     sH   j  | |d j  }|d r tdj d|d  |d S )N)r   rC   r<   r  zRPC error on z: r  )r  r  r  r   r  )rC   r<   responser  r   r   r   call  s   
z'_ZmqRpcHandle.__getattr__.<locals>.callr   r   r  r  r   r  r   __getattr__  s   z_ZmqRpcHandle.__getattr__N)r!   rN   rO   r   rQ   ru   r  r   r   r   r   r    s    r  c                   @   s0   e Zd ZdZdefddZdee fddZdS )	_RpcBroadcastBasez9Base for broadcasting method calls to dumper instance(s).r  c                 C   s   t r   )NotImplementedError)r   r  r   r   r   r  "  s   z_RpcBroadcastBase.__getattr__r  c                 C   
   || _ d S r   _handlesr   r  r   r   r   ru   %     
z_RpcBroadcastBase.__init__N)	r!   rN   rO   r   rQ   r  r   r  ru   r   r   r   r   r    s    r  c                   @   s(   e Zd ZdZd
ddZdefddZd	S )rs   zJCalls methods directly on the local dumper, wrapping the result in a list.dumperri   c                 C   r  r   )_dumper)r   r  r   r   r   ru   ,  r  z_LocalOnlyBroadcast.__init__r  c                    r  )Nc                     s   t j | i |gS r   )r"   r  rC   r<   r  r   r   r  0  r   z-_LocalOnlyBroadcast.__getattr__.<locals>.callr   r  r   r  r   r  /  s   z_LocalOnlyBroadcast.__getattr__N)r  ri   )r!   rN   rO   r   ru   rQ   r  r   r   r   r   rs   )  s    
rs   c                   @   s0   e Zd ZdZdee fddZdefddZdS )	r  ztBroadcasts method calls to all ZMQ RPC handles.

    Returns a list of results, one per rank (ordered by rank).
    r  c                 C   r  r   r  r  r   r   r   ru   <  r  z_ZmqRpcBroadcast.__init__r  c                    r  )Nc                     s    fddj D S )Nc                    s    g | ]}t | i qS r   )r"   )r5   handle)rC   r<   r  r   r   
<listcomp>A  s    z>_ZmqRpcBroadcast.__getattr__.<locals>.call.<locals>.<listcomp>r  r  r  r  r   r  @  s   z*_ZmqRpcBroadcast.__getattr__.<locals>.callr   r  r   r  r   r  ?  s   z_ZmqRpcBroadcast.__getattr__N)	r!   rN   rO   r   r   r  ru   rQ   r  r   r   r   r   r  6  s    r#   r4   c                 C   s@   t | }|d u s| s|S zt|W S  ty   | Y S w r   )r8   r9   rH   rK   r   )r#   r4   r*   r   r   r   r   L  s   

r   c                  C   s   t  t jt j} z| d |  d W S  ty   Y nw zt  }t |}|r5|dkr5|dkr5|W S W n	 ty?   Y nw zt  t jt j} | d |  d W S  tyb   t	d Y d S w )N)z8.8.8.8P   r   z	127.0.0.1rn  )z2001:4860:4860::8888r  zCan not get local ip by remote)
r  AF_INET
SOCK_DGRAMr  getsocknamer   gethostnamegethostbynameAF_INET6ry   )shostnameipr   r   r   r  V  s0   



r  )rj   c                 C   s`   | d u rd S t | trdd | D S t | tjs| S |  dk r"| S dd | jD }| t| S )Nc                 S   s   g | ]}t |qS r   )r   )r5   r  r   r   r   r    s    z'get_truncated_value.<locals>.<listcomp>rs  c                 S   s&   g | ]}|d krt ddnt dqS )2   r      N)slice)r5   dim_sizer   r   r   r    s   & )r%   tupler   r   numelr   )r*   slicesr   r   r   r   |  s   
r   c                 C   s
  t | tjsdt|  d|  S |  dkr|   nd}|  dkr*|   nd}|  dkr8|   nd}tj	dd t
|  dd }t
|  dd }tj	d	d dt|  d
| j d| j d| j d|   d| j d| d| d| d| d| S )zC
    from sglang.srt.debug_utils.dumper import get_tensor_info
    ztype=z value=r   N
   )	precisionr     r   r   r   z stride=z
 req_grad=z min=z max=z mean=z x_sample_head=z x_sample_tail=)r%   r   r   r    r  floatminmaxmeanset_printoptionsrQ   flattenr   r   r   strider   )r  r  r  r  x_sample_headx_sample_tailr   r   r   get_tensor_info  s>   	
r  )r`   )r   )Frx  r8   r   r  r   rz   abcr   r   
contextlibr   copyr   dataclassesr   r   r   r	   	functoolsr
   http.serverr   r   pathlibr   typingr   r   r   r   r   r   r   r   torch.distributeddistributedr   r   rS   ri   rQ   r   rK   r   r   r   r   r  r   r   r   r   r   r   r  r  objectr   ro  r   r  r  rs   r  r   r  r7   r  r   r  r   r   r   r   <module>   sn    $B"  X	2&#
4

 