o Ti@sLddlmZGdddeZddZddZdd d Zdd d ZddZd S))partition_datac@sFeZdZddZddZddZddd Zd d Zd d ZddZ dS)meg_2d_parallel_mapcCs||_||_i|_dSN) pp_degree tp_degreemap)selfrrr W/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/checkpoint/reshape_meg_2d.py__init__ s zmeg_2d_parallel_map.__init__cs$fddtjjD_dS)Ncs(i|]}|j|j|gqSr ) _make_keyr).0irr r sz3meg_2d_parallel_map.simple_init..)rangerrrrr rr simple_inits  zmeg_2d_parallel_map.simple_initcCsV|||t|tusJ|||}||jvr g|j|<|j||7<dSr)_validate_indicestypelistr rkeys)rpp_indextp_indexdatakeyr r r add_datas   zmeg_2d_parallel_map.add_dataNcCsv||||durtt|jn|g}|durtt|jn|g}g}|D]}|D] }||j|||7}q*q&|Sr)rrrrrrr )rrr pp_indices tp_indicesresultrjr r r get_datas zmeg_2d_parallel_map.get_datacCs4t||jD] \}}t|d|q dS)Nz = )printritems)rtagrvaluer r r print_data+s zmeg_2d_parallel_map.print_datacCs4|dus ||jks J|dus||jksJdSdSr)rr)rrrr r r r0sz%meg_2d_parallel_map._validate_indicescCs|d|S)N,r )rrrr r r r 4szmeg_2d_parallel_map._make_key)NN) __name__ __module__ __qualname__r rrr r%rr r r r r r s   rcCsZ|j}t||}t|D]}|j|dd}t||}t|D] }|||||qq |SN)rr)rrrr rr) old_2d_map new_tp_degree old_pp_degree new_2d_maprranks_for_pp_index split_ranksrr r r _reshape_tp_dimension8    r1cCsZ|j}t||}t|D]}|jd|d}t||}t|D] }|||||qq |Sr*)rrrr rr)r+ new_pp_degree old_tp_degreer.rranks_for_tp_indexr0rr r r _reshape_pp_dimensionDr2r6FcCs||ksJ||ks Jt||}||r|d||kr&t||}n|}|r/|d||kr9t||}n|}|rB|d|S)Nzoriginal_2d_map:zafter_tp_reshape:z final_2d_map:)rrr%r1r6)r-r4r3r,verboser+ new_tp_map final_mapr r r reshape_meg_2d_parallelPs"        r:Nc sF|||}td|d|d|d|t||}t||}|||}||}||} ||} g} t|D]"| } d| } t|D]}t| || |}| t|qHq8td| g}t|D]fdd| D}|t|qftd |g}t|D]t|d|}|t|qtd |||| fS) a Initialize model data parallel groups. Arguments: tp_size: number of GPUs used to parallelize model tensor. pp_size: number of GPUs used to parallelize model pipeline. dp_size: number of GPUs used to parallelize model data. Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize the model pipeline. The present function will create 8 tensor model-parallel groups, 4 pipeline model-parallel groups and 8 data-parallel groups as: 8 data_parallel groups: [g0, g2], [g1, g3], [g4, g6], [g5, g7], [g8, g10], [g9, g11], [g12, g14], [g13, g15] 8 tensor model-parallel groups: [g0, g1], [g2, g3], [g4, g5], [g6, g7], [g8, g9], [g10, g11], [g12, g13], [g14, g15] 4 pipeline model-parallel groups: [g0, g4, g8, g12], [g1, g5, g9, g13], [g2, g6, g10, g14], [g3, g7, g11, g15] Note that for efficiency, the caller should make sure adjacent ranks are on the same DGX box. For example if we are using 2 DGX-1 boxes with a total of 16 GPUs, rank 0 to 7 belong to the first box and ranks 8 to 15 belong to the second box. z *** tp=z, pp=z, dp=z, world=rDPcsg|]}|qSr r )r data_parallel_group_ranksrr r sz!get_mpu_ranks..PPTP)r!minrappendr)tp_sizepp_sizedp_sizevirtual_pp_size world_sizetensor_model_parallel_sizepipeline_model_parallel_sizedata_parallel_size num_tensor_model_parallel_groups"num_pipeline_model_parallel_groupsnum_data_parallel_groupsall_dp_group_ranks start_rankend_rankrranksall_pp_group_ranksall_tp_group_ranksr r=r get_mpu_ranksks8              rTcCstd|d||\}}}|\}}}t|||d\}} } t|||d\} } } t|||d\}}}tdt|D]\}}t||d| |q:tdt| D]\}}t| |d||qTdS)zo reshape([tp_size_src, pp_size_src, dp_size_src], [tp_size_tgt, pp_size_tgt, dp_size_tgt]) z *** Reshaping: z => )rCrDrEz *** TP contraction:z *** PP contraction:N)r!rT enumerate)srctgt tp_size_src pp_size_src dp_size_src tp_size_tgt pp_size_tgt dp_size_tgt tp_ranks1 pp_ranks1 dp_ranks1 tp_ranks2 pp_ranks2 dp_ranks2 tp_ranks3 pp_ranks3 dp_ranks3rrr r r reshapes  rh)F)rrrN) reshape_utilsrobjectrr1r6r:rTrhr r r r s /  K