o
    
۾iJ                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ eeZe jZe jZe jZG d	d
 d
e jZe jZe jZe jZG dd dZe jZG dd dZeG dd dZ G dd dZ!g dZ"dS )    N)	dataclass)Any)ReduceOp)envs)init_logger)current_platform)find_nccl_libraryc                   @   s   e Zd Zdejd fgZdS )ncclUniqueIdinternal   N)__name__
__module____qualname__ctypesc_byte_fields_ r   r   h/home/ubuntu/.local/lib/python3.10/site-packages/vllm/distributed/device_communicators/pynccl_wrapper.pyr	   1   s    r	   c                   @   sh   e Zd ZdZdZdZdZdZdZdZ	dZ
dZdZdZdZd	Zd	Zd
ZdZdZedejdefddZdS )ncclDataTypeEnumr                           	   
      dtypereturnc                 C   s   |t jkr| jS |t jkr| jS |t jkr| jS |t jkr | jS |t j	kr(| j
S |t jkr0| jS |t jkr8| jS |t jkr@| jS |t krI| jS td| d)NzUnsupported dtype z^: should be one of int8, uint8, int32, int64, float16, float32, float64, bfloat16, float8e4m3.)torchint8ncclInt8uint8	ncclUint8int32	ncclInt32int64	ncclInt64float16ncclFloat16float32ncclFloat32float64ncclFloat64bfloat16ncclBfloat16r   	fp8_dtypencclFloat8e4m3
ValueError)clsr    r   r   r   
from_torchN   s*   








zncclDataTypeEnum.from_torchN)r   r   r   r$   ncclCharr&   r(   ncclInt
ncclUint32r*   
ncclUint64r,   ncclHalfr.   	ncclFloatr0   
ncclDoubler2   r4   ncclNumTypesclassmethodr"   r    intr7   r   r   r   r   r   ;   s(    r   c                   @   s:   e Zd ZdZdZdZdZdZdZe	de
defd	d
ZdS )ncclRedOpTypeEnumr   r   r   r   r   r   opr!   c                 C   s^   |t jkr| jS |t jkr| jS |t jkr| jS |t jkr | jS |t j	kr(| j
S td| )NzUnsupported op: )r   SUMncclSumPRODUCTncclProdMAXncclMaxMINncclMinAVGncclAvgr5   )r6   rC   r   r   r   r7   t   s   




zncclRedOpTypeEnum.from_torchN)r   r   r   rE   rG   rI   rK   rM   
ncclNumOpsr@   r   rA   r7   r   r   r   r   rB   l   s    rB   c                   @   s*   e Zd ZU eed< eed< ee ed< dS )FunctionnamerestypeargtypesN)r   r   r   str__annotations__r   listr   r   r   r   rO      s   
 rO   c                   @   s  e Zd ZU edejegedeeejgedeee	gedeee
eje	ejgedeeeejeee
egedeeeejeeeje
egedeeeejee
egedeeeejeee
eged	eeejeeje
eged
eeejeeje
egedeeeejeeje
egedee
gedeg edeg edee
eejeeejgedee
eggZi Zeeef ed< i Zeeeeef f ed< dQdedB fddZdedefddZdeddfddZdefddZdefdd Zde	fd!d"Zd#ede	fd$d%Z d&ed'e	d(ede
fd)d*Z!d+ed,ed-ed.ed/ed0e
d1eddfd2d3Z"d+ed,ed-ed.ed/ed4ed0e
d1eddfd5d6Z#d+ed,ed-ed.ed/ed0e
d1eddfd7d8Z$d+ed,ed-ed.ed0e
d1eddfd9d:Z%d+ed-ed.ed;ed0e
d1eddfd<d=Z&d,ed-ed.ed>ed0e
d1eddfd?d@Z'd+ed,ed-ed.ed4ed0e
d1eddfdAdBZ(d0e
ddfdCdDZ)dRdEdFZ*dRdGdHZ+d0e
dIedJedKedef
dLdMZ,d0e
dNeddfdOdPZ-dS )SNCCLLibraryncclGetErrorStringncclGetVersionncclGetUniqueIdncclCommInitRankncclAllReduce
ncclReducencclAllGatherncclReduceScatterncclSendncclRecvncclBroadcastncclCommDestroyncclGroupStartncclGroupEndncclCommWindowRegisterncclCommWindowDeregisterpath_to_library_cachepath_to_dict_mappingNso_filec              
   C   s  |pt  }z|tjvrt|}|tj|< tj| | _W n ty3 } zt	d|t

  |d }~ww |tjvrzi }tjD ]6}zt| j|j}|j|_|j|_|||j< W q> tyt   |jdv rstjrmtd|j| t rsY q> w |tj|< tj| | _d S )NaI  Failed to load NCCL library from %s. It is expected if you are not running on NVIDIA/AMD GPUs.Otherwise, the nccl library might not exist, be corrupted or it does not support the current platform %s. If you already have the library, please set the environment variable VLLM_NCCL_SO_PATH to point to the correct nccl library path.)re   rf   zThe symbol %s is not found in the NCCL library %s. To enable VLLM_USE_NCCL_SYMM_MEM  please update your NCCL version to >= 2.27.03.)r   rV   rh   r   CDLLrg   lib	Exceptionloggererrorplatformexported_functionsgetattrrP   rQ   rR   AttributeErrorr   VLLM_USE_NCCL_SYMM_MEMwarning_oncer   is_rocm_funcs)selfri   rk   erv   funcfr   r   r   __init__;  sJ   







zNCCLLibrary.__init__resultr!   c                 C   s   | j d |dS )NrW   zutf-8)rv   decode)rw   r|   r   r   r   rW   o  s   zNCCLLibrary.ncclGetErrorStringc                 C   s$   |dkr|  |}td| d S )Nr   zNCCL error: )rW   RuntimeError)rw   r|   	error_strr   r   r   
NCCL_CHECKr  s   
zNCCLLibrary.NCCL_CHECKc                 C   s(   t  }| | jd t | |jS )NrX   )r   c_intr   rv   byrefvalue)rw   versionr   r   r   ncclGetRawVersionw  s   zNCCLLibrary.ncclGetRawVersionc                 C   sR   t |  }|d d}|dd d}|dd  d}| d| d| S )Nr   0r   r   .)rS   r   lstrip)rw   version_strmajorminorpatchr   r   r   rX   }  s
   zNCCLLibrary.ncclGetVersionc                 C   s$   t  }| | jd t| |S )NrY   )r	   r   rv   r   r   )rw   	unique_idr   r   r   rY     s   zNCCLLibrary.ncclGetUniqueIddatac                 C   s@   t |dkrtdt | dt }tt|j|d |S )Nr   z)Expected 128 bytes for ncclUniqueId, got z bytes)lenr5   r	   r   memmove	addressofr
   )rw   r   r   r   r   r   unique_id_from_bytes  s   z NCCLLibrary.unique_id_from_bytes
world_sizer   rankc                 C   s*   t  }| | jd t|||| |S )NrZ   )
ncclComm_tr   rv   r   r   )rw   r   r   r   commr   r   r   rZ     s   zNCCLLibrary.ncclCommInitRanksendbuffrecvbuffcountdatatyperC   r   streamc              
   C   $   |  | jd ||||||| d S )Nr[   r   rv   rw   r   r   r   r   rC   r   r   r   r   r   r[     
   zNCCLLibrary.ncclAllReducerootc	           	      C   s&   |  | jd |||||||| d S )Nr\   r   )	rw   r   r   r   r   rC   r   r   r   r   r   r   r\     s
   zNCCLLibrary.ncclReducec              
   C   r   )Nr^   r   r   r   r   r   r^     r   zNCCLLibrary.ncclReduceScatterc              	   C   "   |  | jd |||||| d S )Nr]   r   )rw   r   r   r   r   r   r   r   r   r   r]     s
   zNCCLLibrary.ncclAllGatherdestc              	   C   r   )Nr_   r   )rw   r   r   r   r   r   r   r   r   r   r_        	zNCCLLibrary.ncclSendsrcc              	   C   r   )Nr`   r   )rw   r   r   r   r   r   r   r   r   r   r`     r   zNCCLLibrary.ncclRecvc              
   C   r   )Nra   r   )rw   r   r   r   r   r   r   r   r   r   r   ra     s
   
zNCCLLibrary.ncclBroadcastc                 C   s   |  | jd | d S )Nrb   r   )rw   r   r   r   r   rb     s   zNCCLLibrary.ncclCommDestroyc                 C      |  | jd   d S )Nrc   r   rw   r   r   r   rc        zNCCLLibrary.ncclGroupStartc                 C   r   )Nrd   r   r   r   r   r   rd   !  r   zNCCLLibrary.ncclGroupEndbuffsize	win_flagsc              	   C   s,   t  }| | jd |||t|| |S )Nre   )ncclWindow_tr   rv   r   r   )rw   r   r   r   r   windowr   r   r   re   $  s   z"NCCLLibrary.ncclCommWindowRegisterr   c                 C   s   |  | jd || d S )Nrf   r   )rw   r   r   r   r   r   rf   /  s   z$NCCLLibrary.ncclCommWindowDeregister)N)r!   N).r   r   r   rO   r   c_char_pncclResult_tPOINTERr   r	   r   buffer_typec_size_tncclDataType_tncclRedOp_tcudaStream_tr   rp   rg   dictrS   r   rT   rh   r{   rW   r   rA   r   rX   rY   bytesr   rZ   r[   r\   r^   r]   r_   r`   ra   rb   rc   rd   re   rf   r   r   r   r   rV      s   
 

  +4	
	
	

	



	



rV   )rV   r   rB   r	   r   r   r   )#r   ro   dataclassesr   typingr   r"   torch.distributedr   vllmr   vllm.loggerr   vllm.platformsr   vllm.utils.ncclr   r   rm   r   r   c_void_pr   r   	Structurer	   r   r   r   r   r   rB   rO   rV   __all__r   r   r   r   <module>   s8   .   ,