o "i} @sUddlZddlZddlZddlmZmZmZmZmZm Z ddl Z ddl m Z ddl mZddgZe sQddlZGdd d Zd d Zeejd _eejd _dSdd lmZmZmZmZmZmZmZmZmZm Z e!e"Z#erzddl$m%Z%Wne&ye#'dYnwGdddej(Z)e)Z*e)e+d<dde,fddZ-GdddZddde,dee.dfdeee,dfdefddZdS)N)DictListOptionalTuple TYPE_CHECKINGUnion) is_available)not_noneinit_device_mesh DeviceMeshc@s eZdZdS)_DeviceMeshStubN)__name__ __module__ __qualname__rr[/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/distributed/device_mesh.pyr sr cCsdSNrrrrr_init_device_mesh_stubsrztorch.distributed.device_mesh) _find_pg_by_ranks_and_tag_get_default_group_get_group_tagget_process_group_ranksget_rankget_world_sizeinit_process_groupis_initialized new_group ProcessGroup) ArrayLikezCDeviceMesh requires numpy >= 1.21 to be installed for type checkingc @seZdZdddZd ddZddd eed fddfd d Zd ddedfddZ d ddee fddZ e dede fddZ e dede fddZd ddede fddZ d!de dedeejddfddZdS)"_MeshEnvreturnNcCsg|_i|_i|_dSr) mesh_stackchild_to_parent_mappingmesh_dim_group_optionsselfrrr__init__?sz_MeshEnv.__init__r cCs t|jdkr td|jdS)Nrz#No device mesh is currently active!)lenr" RuntimeErrorr%rrrget_current_meshFs z_MeshEnv.get_current_mesh parent_meshsubmesh_dim_names.c sfdd|D}fdd|D}ttjj}|D]}||qjjg||Rjdg|R}}|D]} tj | |dd} || vrM| } q<| _ fdd|D| _ |j | <| S)Ncsg|] }tj|qSr)r mesh_dim_namesindex.0 mesh_dim_namer,rr Osz._MeshEnv.create_child_mesh..csg|]}j|qSr)meshsizer1mesh_dimr3rrr4Ss r(Fr. _init_backendcsg|]}j|qSr)_dim_group_infosr7r3rrr4ns ) listranger5ndimremovepermutereshaperr device_type _parent_meshr;r#) r&r,r- submesh_dimssubmesh_dim_sizesmesh_dims_remained submesh_dimpg_ranks_by_dimcur_rankmesh_ndsubmesh res_submeshrr3rcreate_child_meshKsF     z_MeshEnv.create_child_mesh device_meshcCs|j|dSr)r#get)r&rNrrrget_parent_meshusz_MeshEnv.get_parent_meshcCsD||}|j}|r |r t|dksJd|d}|||SdS)z Return the index of the mesh dim in the parent mesh. The device_mesh passed in needs to be sliced out from a parent mesh. z%The child mesh can only be a 1D mesh.rN)rPr.r)get_mesh_dim_by_name)r&rNr,child_mesh_dim_nameschild_mesh_dim_namerrrget_parent_mesh_dimxs  z_MeshEnv.get_parent_mesh_dimrBcCs t|Sr)_get_device_handle device_countrBrrrnum_devices_per_host z_MeshEnv.num_devices_per_hostcCstt|Sr)rr rYrXrrr num_hostssz_MeshEnv.num_hostsr2cCsT|jdus t|jdkrtd||jvr"td|dd|jt|j|S)NrzNo `mesh_dim_names` found.zMesh dimension 'z' does not exist.z.Available mesh dimensions are: mesh_dim_names=)r.r)KeyErrorr r/)r&rNr2rrrrRs    z_MeshEnv.get_mesh_dim_by_namedimbackend pg_optionscCs||f|j|<dSr)r$)r&r]r^r_rrr_set_mesh_dim_group_optionssz$_MeshEnv._set_mesh_dim_group_optionsr!Nr!r r)rrrr'r+rstrrMrrPintrU staticmethodrYr[rRrOptionsr`rrrrr >sB    * r _mesh_resourcescudarBcCs tt|dS)a: Get the module corresponding to the device_type which is cuda or cuda-like device. For example, when the device_type is cuda, the module `torch.cuda` is returned. Return None when there is no corresponding module for device_type, otherwise return the corresponding module. N)getattrtorchrXrrrrVs rVc@seZdZUdZeed<ejed<ee edfed<ddddede ejd fdee edfd e d df d d Z ddZ ddZd4ddZd5ddZd efddZddZded e fddZde ee edffd dfddZd6dee eefd efd d!Zd eefd"d#Ze d6dd$d%e eeefdedee ejd fdee edfd df d&d'Zd6deed efd(d)Zed efd*d+Zed e edffd,d-Z d efd.d/Z!d6dee eefd efd0d1Z"d eeefd2d3Z#dS)7r a DeviceMesh represents a mesh of devices, where layout of devices could be represented as a n-d dimension array, and each value of the n-d dimensional array is the global id of the default process group ranks. DeviceMesh could be used to describe the layout of devices across the cluster, and serves as a proxy for communication among the device lists within the cluster. DeviceMesh can be used as a context manager. .. note:: DeviceMesh follows SPMD programming model, which means the same PyTorch Python program is running on all processes/ranks in the cluster. Therefore, users need to make sure the `mesh` array (which describes the layout of devices) should be identical across all ranks. Inconsistent `mesh` will lead to silent hang. Args: device_type (str): The device type of the mesh. Currently supports: "cpu", "cuda/cuda-like". mesh (ndarray): A multi-dimensional array or an integer tensor describing the layout of devices, where the IDs are global IDs of the default process group. Returns: DeviceMesh: A :class:`DeviceMesh` object representing the device layout. The following program runs on each process/rank in an SPMD manner. In this example, we have 2 hosts with 4 GPUs each. A reduction over the first dimension of mesh will reduce across columns (0, 4), .. and (3, 7), a reduction over the second dimension of mesh reduces across rows (0, 1, 2, 3) and (4, 5, 6, 7). Example:: >>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import DeviceMesh >>> >>> # Initialize device mesh as (2, 4) to represent the topology >>> # of cross-host(dim 0), and within-host (dim 1). >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]]) rBr5.r.NTr9rr:r!cCs||_t|tjr|jjdkrtd|t|tjr%|jtj dntj |dtj d|_ |r5t |nd|_ t |j |_d|_t|_|dkr{|rX|||j tk}|ddvsiJ|ddkrv|dnd|_dSdS)Ncpuz!`mesh` must be a CPU tensor, got dtypedevicermxlar)rrQ)rB isinstancerjTensorrotype ValueErrordetachtordtensorr5tupler.flattentolist_flatten_mesh_listrC threading get_ident _thread_id_get_or_create_default_group_init_process_groupsrnonzeror6_coordinate_on_dim)r&rBr5r.r: rank_coordsrrrr's(  zDeviceMesh.__init__cCst}|stt}|j|krtd|jdt|j}|sK|rK|}||krC||dkrCtd|d|d|jd| t |t S)Nz=Mesh should not be bigger than default world size, but found z ranks!rz8DeviceMesh only support homogeneous hardware, but found z ranks and  z devices!) rrrr5numelr*rVrBrW set_devicerr)r&default_initialized world_size device_handlerYrrrr s0  z'DeviceMesh._get_or_create_default_groupc Cs g}|jjdkr#|jtkr#|ttttttj fn^t|jjD]W}|j d| d|j |}|D]C}| }|tjvrOtj|\}}nd\}}t|||d}||vrt||krrtd|jd|d|tt|||j fqrrappendrrr<r= group_nameswapdimsrAr6rzrgr$rrr)r*r r;) r&dim_group_infosr]rHdim_meshsubgroup_ranksr^r_ dim_grouprrrr&sR         $zDeviceMesh._init_process_groupscCstj||Sr)rgr"rr%rrr __enter__js zDeviceMesh.__enter__cCstjdSr)rgr"pop)r&exc_type exc_value exc_tracebackrrr__exit__pszDeviceMesh.__exit__cCs:|jsd|jd}|Sd|jd|jd}|S)Nz DeviceMesh()z, mesh_dim_names=)r.r5rz)r&device_mesh_reprrrr__repr__tszDeviceMesh.__repr__cCs>t|dd|_|jst|j|jj|j|j|j|j f|_|jS)N_hash) rirhashr{r5shaperBr.rCr~r%rrr__hash__|s zDeviceMesh.__hash__othercCsnt|tsdSt|t|krdS|j|jko6|jj|jjko6|j|jko6|j|jko6|j|jko6|j |j kS)NFT) rqr idr{r5rrBr.rCr~)r&rrrr__eq__s      zDeviceMesh.__eq__csjstdt|tr|fn|}d|djd}|jkr"St|tjks6tfdd|Ds:t||d}j|}t|j|t|D] \}}||kr\t|qPt |}|S)a Slice the current DeviceMesh based on the mesh_dim_name given to create a child DeviceMesh. Args: mesh_dim_name (Union[str, Tuple[str]]): the name or the tuple of names of the mesh dimension of the parent DeviceMesh to create the child DeviceMesh for. Returns: A :class:`DeviceMesh` object The following program runs on each process/rank in an SPMD manner. In this example, we have 2 hosts with 4 GPUs each. Calling mesh["tp"] on rank 0, 1, 2, 3 would return a 1D child DeviceMesh:([0, 1, 2, 3]). Calling mesh["tp"] on rank 4, 5, 6, 7 would return a 1D child DeviceMesh:([4, 5, 6, 7]). Calling mesh["dp"] on rank 0, 4 would return a 1D child DeviceMesh:([0, 4]). Calling mesh["dp"] on rank 1, 5 would return a 1D child DeviceMesh:([1, 5]). Calling mesh["dp"] on rank 2, 6 would return a 1D child DeviceMesh:([2, 6]). Calling mesh["dp"] on rank 3, 7 would return a 1D child DeviceMesh:([3, 7]). Example:: >>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import DeviceMesh >>> >>> # Initialize device mesh as (2, 4) to represent the topology >>> # of cross-host(dim 0), and within-host (dim 1). >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]]) z1Cannot slice a DeviceMesh without mesh_dim_names!zInvalid mesh_dim_name zG specified. Valid mesh_dim_names should be a contiguous subsequence of .c3s|]}|jvVqdSrr.r0r%rr s  z)DeviceMesh.__getitem__..r) r.r*rqrcr)allr\r/ziprgrM)r&r. error_msgoutermost_dim_nameoutermost_dim_idxijrKrr%r __getitem__s2    zDeviceMesh.__getitem__r8cCst|ds td|jjdkr|durtd|jjddd|jjdkr,|dur,d }n t|tr7t||n|}tt |j |dd S) a Returns the single ProcessGroup specified by mesh_dim, or, if mesh_dim is not specified and the DeviceMesh is 1-dimensional, returns the only ProcessGroup in the mesh. Args: mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index of the mesh dimension. Default is None. Returns: A :class:`ProcessGroup` object. r;z*DeviceMesh process groups not initialized!rQNFound the DeviceMesh have dimensionsJOptional kwarg `mesh_dim` needs to be specified when device_mesh.ndim > 1.zmIf you want to get the list of all the ProcessGroups in the DeviceMesh,please use `get_all_groups()` instead.rr ) hasattrr*r5r>rqrcrgrRr rr;r&r8rrr get_groups" zDeviceMesh.get_groupcsfddtjjDS)z Returns a list of ProcessGroups for all mesh dimensions. Returns: A list of :class:`ProcessGroup` object. csg|]}|qSr)r)r1rr%rrr4sz-DeviceMesh.get_all_groups..)r=r5r>r%rr%rget_all_groupsszDeviceMesh.get_all_groupsrgroupcCs:t|trFt|}t|tjr||ks|dur)||kr)tdt|d|tj|dtj d}t |||dd}t |||j fg|_ |St|}t|dkrTtd |dur\td t|tjrl|jtj dd ntj|dtj d}|jt|krtd |d t|dt |||dd}dd|D|_ |S)aN Contstructs a :class:`DeviceMesh` with ``device_type`` from an existing :class:`ProcessGroup`. The constructed device mesh has number of dimensions equal to the number of groups passed. If more than one group is passed, then the ``mesh`` argument is required. Nz Invalid mesh z for ProcessGroup with ranks rkrnFr9rz.Expects at least one ProcessGroup to be passedz0Must pass mesh if passing multiple ProcessGroups)rmrozEExpects mesh with ndim equal to number of ProcessGroups but got mesh z and z ProcessGroupscSs g|] }t|t||jfqSr)rrr)r1rrrrr4<sz)DeviceMesh.from_group..)rqrrrjrrrzrtrcrwrdr rrr;r<r)rurvr>)rrBr5r. group_ranksrNgroupsrrr from_groupsV     zDeviceMesh.from_groupcCs|dur |jS|j|Sr)r5rr6rrrrr6FszDeviceMesh.sizecCs|jjSr)r5r>r%rrrr>IszDeviceMesh.ndimcCs t|jjSr)rxr5rr%rrrrMrZzDeviceMesh.shapecCstS)z: Returns the current global rank. )rr%rrrrQszDeviceMesh.get_rankcCs`|jdkr|durtd|jjdd|durd}t||}t|ts*Jdtt|S)a{ Returns the local rank of the given mesh_dim of the DeviceMesh. Args: mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index of the mesh dimension. Default is None. Returns: An integer denotes the local rank. The following program runs on each process/rank in an SPMD manner. In this example, we have 2 hosts with 4 GPUs each. Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 0, 1, 2, 3 would return 0. Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 4, 5, 6, 7 would return 1. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 0, 4 would return 0. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 1, 5 would return 1. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 2, 6 would return 2. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 3, 7 would return 3. Example:: >>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import DeviceMesh >>> >>> # Initialize device mesh as (2, 4) to represent the topology >>> # of cross-host(dim 0), and within-host (dim 1). >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]]) rQNrrrrz1We expect ProcessGroup before calling `get_rank`!)r>r*r5r rrqrr)r&r8mesh_dim_grouprrrget_local_rankWs zDeviceMesh.get_local_rankcCs|jr|jSdS)z Return the relative indices of this rank relative to all dimensions of the mesh. If this rank is not part of the mesh, return None. N)rr%rrrget_coordinateszDeviceMesh.get_coordinaterbrar)$rrr__doc__rc__annotations__rjrrrrrboolr'rrrrrrobjectrrrdrrrrrerr6propertyr>rrrrrrrrr sn '   ( D  ?$  > *r mesh_shape.r.r!cCs|dur.tt|t|krtdd|t|t|kr.tddt|dt|d|r=|s=td|d d td tjt|tj d  |}Wdn1s[wYt |||d }|S)a Initializes a `DeviceMesh` based on `device_type`, `mesh_shape`, and `mesh_dim_names` parameters. This creates a DeviceMesh with an n-dimensional array layout, where `n` is the length of `mesh_shape`. If `mesh_dim_names` is provided, each dimension is labeled as `mesh_dim_names[i]`. .. note:: `init_device_mesh` follows SPMD programming model, meaning the same PyTorch Python program runs on all processes/ranks in the cluster. Ensure `mesh_shape` (the dimensions of the nD array describing device layout) is identical across all ranks. Inconsistent `mesh_shape` may lead to hanging. .. note:: If no process group is found, init_device_mesh will initialize distributed process group/groups required for distributed communications behind the scene. Args: device_type (str): The device type of the mesh. Currently supports: "cpu", "cuda/cuda-like". Passing in a device type with a GPU index, such as "cuda:0", is not allowed. mesh_shape (Tuple[int]): A tuple defining the dimensions of the multi-dimensional array describing the layout of devices. mesh_dim_names (Tuple[str], optional): A tuple of mesh dimension names to assign to each dimension of the multi-dimensional array describing the layout of devices. Its length must match the length of `mesh_shape`. Each string in `mesh_dim_names` must be unique. Returns: DeviceMesh: A :class:`DeviceMesh` object representing the device layout. Example:: >>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import init_device_mesh >>> >>> mesh_1d = init_device_mesh("cuda", mesh_shape=(8,)) >>> mesh_2d = init_device_mesh("cuda", mesh_shape=(2, 8), mesh_dim_names=("dp", "tp")) Nz"Each mesh_dim_name must be unique.z/Found repeated mesh_dim_name in mesh_dim_names z6mesh_shape and mesh_dim_names should have same length!zFound len(mesh_dim_names): z and len(mesh_shape):rz4Device type with GPU index is not supported but got z. zUIf you maintained a 'torch.device' object, it's recommended to pass in 'device.type'.rkrl)rBr5r.) r)setr*isalpharjroarangemathprodrdviewr )rBrr.r5rNrrrr s2)   )rh)/loggingrr|typingrrrrrrrjtorch.distributedrutils._typing_utilsr __all__sysr rmodulesr r "torch.distributed.distributed_c10drrrrrrrrrr getLoggerrlogger numpy.typingr ImportErrorwarninglocalr rgrrcrVrdrrrrsZ     0  l Z