o
    پi"                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZmZmZmZmZ d dlmZ e eZG dd deZd	d
 Zdd ZdS )    N)urlparse)envs)kill_process_tree)!DEFAULT_TIMEOUT_FOR_SERVER_LAUNCHDEFAULT_URL_FOR_TESTCustomTestCaseis_in_cipopen_with_error_check)wait_for_http_readyc                   @   sB   e Zd Zedd Zedd ZeedfddZedd	 ZdS )
PDDisaggregationServerBasec              	   C   s$  t t}|j| _t|j}|| _t|d  | _t|d  | _	d| j d| j | _
d| j d| j	 | _d| j d| j | _td| jd| jd| jd| j	 d	\| _| _| _t rmd
dg| _dt g| _d S d
tj g| _dtj g| _| jd d u rg | _d}t| d S d S )Nd      zhttp://:zcls.base_host=z cls.lb_port=z cls.prefill_port=z cls.decode_port=)NNNz!--disaggregation-transfer-backendmooncakez--disaggregation-ib-device   zJNo RDMA devices specified for disaggregation test, using default settings.)r   r   hostname	base_hoststrportlb_portintprefill_portdecode_portprefill_url
decode_urllb_urlprint
process_lbprocess_decodeprocess_prefillr   transfer_backendget_rdma_devices_argsrdma_devicesr   SGLANG_TEST_PD_DISAGG_BACKENDgetSGLANG_TEST_PD_DISAGG_DEVICESwarningswarn)cls
parsed_url	base_portmsg r,   f/home/ubuntu/.local/lib/python3.10/site-packages/sglang/test/server_fixtures/disaggregation_fixture.py
setUpClass   s2   
(
z%PDDisaggregationServerBase.setUpClassc                 C   sZ   dddddd| j d| jd| jd	| jg}td
t| t|| _| j	| j
d | jd d S )Npython3z-mzsglang_router.launch_routerz--pd-disaggregationz	--mini-lbz	--prefillz--decodez--hostz--portzStarting load balancer:z/health)process)r   r   r   r   r   shlexjoinr	   r   wait_server_readyr   )r(   
lb_commandr,   r,   r-   	launch_lb7   s"   
z$PDDisaggregationServerBase.launch_lbNc                 C   s"   t |||d td| d d S )N)urltimeoutr0   zServer z	 is ready)r
   r   )r(   r6   r7   r0   r,   r,   r-   r3   L   s   z,PDDisaggregationServerBase.wait_server_readyc                 C   sr   | j | j| jfD ])}|r1zt|j W q ty0 } ztd|j d|  W Y d }~qd }~ww qtd d S )NzError killing process z:    )	r   r   r   r   pid	Exceptionr   timesleep)r(   r0   er,   r,   r-   tearDownClassS   s   "z(PDDisaggregationServerBase.tearDownClass)	__name__
__module____qualname__classmethodr.   r5   r   r3   r>   r,   r,   r,   r-   r      s    

r   c                  C   s  d} t j| std|  dS tt | }td| g }|D ]}t j| |dd}t j|s9td| q!d}d	}t j|d
}t j|d}zt|}	|		 
 }W d   n1 saw   Y  W n ttfyr   Y nw z$t|}	|		 
 }
t|
 d }W d   n1 sw   Y  W n ttttfy   Y nw td|||d| v  d| v rtd| q!d| vrtd|| q!|dkr|dk rtd|| q!|| td| q!td|t| |r|S dS )a  Auto-detect available high-speed RDMA devices from sysfs.

    Filters for devices that are:
    1. Not Ethernet NICs (excludes devices with 'eth' in the name like mlx5_eth0)
    2. Active (port state)
    3. High-speed (rate >= 100 Gbps to exclude regular Ethernet NICs)
    z/sys/class/infinibandzIB sysfs path %s does not existNzAll IB devices in sysfs: %sports1zDevice %s: SKIPPED (no port 1)unknownstaterater   z5Device %s: state=%s, rate=%d Gbps, has_eth_in_name=%sethz+Device %s: SKIPPED (contains 'eth' in name)ACTIVEzDevice %s: SKIPPED (state=%s)r   z!Device %s: SKIPPED (rate=%d Gbps)zDevice %s: INCLUDEDz"Filtered IB devices: %s (count=%d))ospathisdirloggerwarningsortedlistdirr2   openreadstripOSErrorIOErrorr   split
ValueError
IndexErrorlowerupperappendlen)ib_sysfs_pathall_devicesdevicesdev	port_pathrG   rH   
state_file	rate_filefrate_strr,   r,   r-   _get_available_ib_devices`   sj   




rg   c               
   C   s  dt fdd} dd }| dpt pdd td	D }td
| t|}td}|s8t	d d
||S zdd |dD }|rKt|dkrSd
||W S W n tyl   t	d|  d
|| Y S w t|d d }|D ]}||  kr|d k sn t	d| d| d|d   qwzdd l}|j }	W n ty   d	}	Y nw |	dkrd	}	||	krtd||	 d
||S td|	| }
td|	||
 g }|D ]}t||
 |d }|||  q|sd
||S d
|S )Nvar_namec                 S   s.   t | }|s	d S dd |dD }|pd S )Nc                 S   s   g | ]
}|  r|  qS r,   )rT   ).0xr,   r,   r-   
<listcomp>   s    zBget_rdma_devices_args.<locals>._parse_list_env.<locals>.<listcomp>,)rK   getenvrW   )rh   valitemsr,   r,   r-   _parse_list_env   s
   
z.get_rdma_devices_args.<locals>._parse_list_envc                 S   s   | d | t | d  gS )Nr      )r]   )rdma_all_devicesr,   r,   r-   _pick_default_pair   s   z1get_rdma_devices_args.<locals>._pick_default_pairSGLANG_CI_RDMA_ALL_DEVICESc                 S   s   g | ]}d | qS )	mlx5_rocer,   )ri   ir,   r,   r-   rk      s    z)get_rdma_devices_args.<locals>.<listcomp>   zResolved rdma_all_devices=%sCUDA_VISIBLE_DEVICESz<CUDA_VISIBLE_DEVICES is not set. Using default RDMA devices.rl   c                 S   s    g | ]}|  rt|  qS r,   )rT   r   )ri   idxr,   r,   r-   rk      s
    
   z%Invalid CUDA_VISIBLE_DEVICES format: z
GPU index z is outside expected group -   r   zDMore RDMA devices (%d) than GPUs (%d), using first and middle devicer   z?GPU-to-RDMA mapping: total_gpus=%d, n_rdma=%d, gpus_per_rdma=%d)r   rg   rangerN   rO   r]   rK   rm   r&   r'   r2   rW   rX   mintorchcudadevice_countr:   maxr\   )rp   rs   rr   n_rdmacuda_visible_devicesgpu_indicesbase_rdma_groupgpu_idxr   
total_gpusgpus_per_rdmar"   	nic_indexr,   r,   r-   r!      s   


r!   )loggingrK   r1   r;   r&   urllib.parser   sglang.srt.environr   sglang.srt.utilsr   sglang.test.test_utilsr   r   r   r   r	   sglang.utilsr
   	getLoggerr?   rN   r   rg   r!   r,   r,   r,   r-   <module>   s    
JL