o ci#@sdZddlZz0ddlZddlmZddlmZddlmZddlmZddlm Z ddlm Z dd lm Z Wn e yAe d wdd l mZmZejejejejejejejejiZejejejejejejejej ej!ej"ej#ej$ej%ejej&ej'ej(ej)ej*ej+ej,ej-ej.ej/i Z0er ddl1Z1ddl2Z1e1j3ej"e1j4ej5e1jeje1j!ej"e1j#ej$e1j%eje1j6eje1j&ej'e1j7ej8e1j(ej)e1j*ej+e1j,ej-e1j.ej/i Z9e:ed rej;e9e1j<<e1j4ej#e1jeje1j!ej!e1j#ej#e1j%ej%e1j6ej%e1j&ej&e1j7ej*e1j(ej(e1j*ej*e1j,ej,i Z=d dZ>ddZ?ddZ@ddZAddZBddZCddZDddZEddZFdd ZGd!d"ZHd#d$ZId%d&ZJd'd(ZKd)d*ZLdS)+z!Code to wrap some NCCL API calls.N)nccl)Device get_versionget_build_versionNcclCommunicator) groupStart)groupEndz*NCCL in Ray requires Cupy being available!)ReduceOptorch_available NCCL_BFLOAT16cCs tjjS)z+Returns the number of compute-capable GPUs.)cupycudaruntimegetDeviceCountrrb/home/ubuntu/.local/lib/python3.10/site-packages/ray/util/collective/collective_group/nccl_util.py get_num_gpusVs rcCtSNrrrrrget_nccl_build_version[rcCrrrrrrrget_nccl_runtime_version_rrcCstSr)r get_unique_idrrrrget_nccl_unique_idcsrcCst|||}|S)a(Create an NCCL communicator using NCCL APIs. Args: world_size: the number of processes of this communicator group. nccl_unique_id: the NCCLUniqueID for this group. rank: the rank of this process. Returns: comm (nccl.ncclComm_t): an NCCL communicator. r) world_sizenccl_unique_idrankcommrrrcreate_nccl_communicatorgs r!cCs|tvr td|t|S)zMap the reduce op to NCCL reduce op type. Args: reduce_op: ReduceOp Enum (SUM/PRODUCT/MIN/MAX). Returns: (nccl.ncclRedOp_t): the mapped NCCL reduce op. z&NCCL does not support reduce op: '{}'.)NCCL_REDUCE_OP_MAP RuntimeErrorformat) reduce_oprrrget_nccl_reduce_opusr&cCsFt|tjr t|jjStrt|tjrt |jSt d t|)z2Return the corresponded NCCL dtype given a tensor.]Unsupported tensor type. Got: {}. Supported GPU tensor types are: torch.Tensor, cupy.ndarray.) isinstancerndarrayNUMPY_NCCL_DTYPE_MAPdtypetyper torchTensorTORCH_NCCL_DTYPE_MAP ValueErrorr$tensorrrrget_nccl_tensor_dtypes     r3cCsBt|tjr |jjStrt|tjrt|jSt d t|)z2Return the corresponded Cupy dtype given a tensor.r') r(rr)r+r,r r-r.TORCH_NUMPY_DTYPE_MAPr0r$r1rrrget_cupy_tensor_dtypes    r5cCs`t|tjr |jjSt|tjr|jStr't|tjr'|j s#t d| St d t|)z@Return the pointer to the underlying memory storage of a tensor.z8Torch tensor must be on GPU when using NCCL collectives.r')r(rr)dataptrnumpyr r-r.is_cudar#data_ptrr0r$r,r1rrrget_tensor_ptrs    r;cCsLt|tjs t|tjr|jStrt|tjrt|St d t |)z*Return the number of elements in a tensor.r') r(rr)r8sizer r-r.numelr0r$r,r1rrrget_tensor_n_elementss   r>cCsFt|tjr t|jStrt|tjrt|St d t |)z)Return the shape of the tensor as a list.r') r(rr)listshaper r-r.r<r0r$r,r1rrrget_tensor_shapes     rAcsPttjrfddjDStrttjrtSt d t )z+Return the strides of the tensor as a list.csg|] }t|jjqSr)intr+itemsize).0strider1rr sz&get_tensor_strides..r') r(rr)stridesr r-r.r?rEr0r$r,r1rr1rget_tensor_stridess    rHc Cst|tjrz|jj}W|Sty}ztd|d}~wwtr7t|tj r7|jj }t|t s5td|St d t|)z!Return the GPU index of a tensor.z!The tensor is not on a valid GPU.Nz!Unsupported tensor type. Got: {}.)r(rr)deviceidAttributeErrorr#r r-r.indexrBr0r$r,)r2rIexecrrrget_tensor_devices     rNcCsd}t|tjrt|tjrt||nRtret|tjr*t|tjr*||n=t|tjrEt|tjrEtjj | }||n"t|tjrbt|tjrbt tjj |}t||nd}nd}|sutdt|t|dS)zCopy the content from src_tensor to dst_tensor. Args: dst_tensor: the tensor to copy from. src_tensor: the tensor to copy to. Returns: None TFzdUnsupported tensor type. Got: {} and {}. Supported GPU tensor types are: torch.Tensor, cupy.ndarray.N)r(rr)copytor r-r.copy_utilsdlpack from_dlpacktoDlpack fromDlpack to_dlpackr0r$r,) dst_tensor src_tensorcopiedtrrr copy_tensors8    r[cCs.t|tstdt|dd|D}|S)zReturns the gpu devices of the list of input tensors. Args: tensors: a list of tensors, each locates on a GPU. Returns: list: the list of GPU devices. zAExpect a list of tensors each locates on a GPU device. Got: '{}'.cSsg|]}t|qSr)rN)rDrZrrrrF$sz*get_tensor_device_list..)r(r?r#r$r,)tensorsdevicesrrrget_tensor_device_lists  r^)M__doc__r8r cupy.cudarrcupy.cuda.ncclrrr r r ImportErrorray.util.collective.typesr r SUMNCCL_SUMPRODUCT NCCL_PRODMINNCCL_MINMAXNCCL_MAXr"int_ NCCL_INT64uint8 NCCL_UINT8uint32 NCCL_UINT32uint64 NCCL_UINT64int8 NCCL_INT8int32 NCCL_INT32int64half NCCL_HALFfloat16 NCCL_FLOAT16float32 NCCL_FLOAT32float64 NCCL_FLOAT64double NCCL_DOUBLEr*r-torch.utils.dlpackboolrBNCCL_INTlongfloat NCCL_FLOATr/hasattrrbfloat16r4rrrrr!r&r3r5r;r>rArHrNr[r^rrrrs            )