o TiL@sddlZddlmZddlmZddlZddlZddlmZddl m Z ddl m Z ddlmZdd Zejjd eed efd d ZeGdddZ  dddZdddZddZdS)N) dataclass)List)Tensor)comm)get_accelerator)loggercCstdkr t|dSdS)Nr)distget_rankrinfo)msgr U/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/zero/mics_utils.py _log_rank0s rtensorsscalecCs|D]}||qdS)N)div_)rrtr r r scale_tensorss rc@s0eZdZdZdZdZdZdZdZdZ dZ dZ dS)MiCS_CommGroupsN) __name__ __module__ __qualname____doc__param_shard_groupparam_shard_sizeparam_shard_rankparam_repli_groupparam_repli_sizeparam_repli_rankparam_intra_node_groupparam_inter_node_shard_groupr r r r r!srFc sttjdt}td|t}|dur"||ks"Jt }t }t |||d}|d} |d} t | dkrVt | dksIJd| dD]} | | gqMt| s^Jd t| sfJd td d | Dt kswJd t | dkrt | t | dksJdt }| D],} t | } || vr| |_t | |_t | |_td|d|jdt j | dq| D]F}t |dkrt |} ||vr| |_t ||_t j | d|_td|d|jdt j | dqd|_d|_d|_td|dq|jt | dksJ|r|d}|dks#Jdt | d|dks0Jt | d|}g}g}| D]>}g}tdt ||D]|||qJg}t|D]fdd |D}||q_||||q>td|td||D]}|D]}t |} ||vr| |_td|qq|D]}|D]}t |} ||vr| |_td|qq|S)z create shard-group, replicate-group from config_file TODO: consider broadcast the config from rank0 Returns: MiCS_CommGroups NDEV_PER_NODEz=creating MiCS communication groups with per node device size N shard_groupsreplicate_groupsrz6replicate groups are empty only for single shard groupz(replicate groups must have the same sizez$shard groups must have the same sizecSsg|]}t|qSr len).0gr r r Vz+create_mics_comm_groups..zall sharded ranks zEnumber of shard groups must equal to the size of each replicate groupzrank z, shard group /)groupz replicate group z replicate group 0/1 span_nodesz>sharding spans on single node, no need for hierarchy allgathercsg|]}|qSr r )r)_gir r r+r,z4create for hierarchy all-gather groups: intra nodes z4create for hierarchy all-gather groups: inter nodes z"create group for intra node ranks z"create group for inter node ranks )intosenvirongetr device_countrrget_data_parallel_grouprget_world_sizer _generate_mics_configr(append_sizes_all_samesum new_grouprrrrr rrr ranger!r") shard_sizedp_grouphierarchical_allgathermpundevices_per_nodegroups world_size global_rankconfigranks_of_shard_groupranks_of_repli_groupr shard_ranks_group repli_ranks n_span_nodesn_gpu_per_nodeintra_node_ranks_groupinter_node_ranks_group shard_group_intra_node_ranks_inter_node_ranks_ranksintra_node_ranksinter_node_ranksr r1r create_mics_comm_groups0s  "                 rYr$c s||dksJ|||dksJd|d|d|di}t|d|}g}t|D]%}|dd|ft}|||fdd td|Dq/||d <||d <t|d||d <|S) aGenerating the configuration for sharding This shard config generation assume that the pipeline stages are partitioned in order, i.e., first ranks hold the stage0, etc. Args: shard_size (int): zero3 data-parallel shard size, FIXME: change the name later pp_size (int): pipeline parallel size, currently, only work with pipeline parallelism + zero rz>dp group size is not dividable by dp_shard_size, (world_size z , pp_size z, dp_shard_size )rNcsg|] }||qSr r )r)jreplicate_sizesame_shard_ranksr r r+sz)_generate_mics_config..r&r%r/)nparangereshaper?tolistr(extend) rF ndev_per_noder@pp_sizerHr%r&r2n_ranksr r\r r:s, $ r:cCs,d}|D]}t|t|dkrdSq|S)zall groups have same lengthTrFr')rEall_samer*r r r r<s r<)FN)r$)r4 dataclassesrtypingrnumpyr_torchr deepspeedrrdeepspeed.acceleratorrdeepspeed.utilsrrjitscriptr3rrrYr:r<r r r r s&       s "