o Tic@s<ddlmZddlZddlZddlmZmZddlmZ ddl m Z ddl m Z ddlmZddlmZddlmZmZmZmZmZmZmZmZmZdd lmZmZmZm Z dd l!m"Z"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)ddl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1e2ej3e4dedddZ5Gddde Z6ddZ7dS)) OrderedDictN)_flatten_dense_tensors_unflatten_dense_tensors)comm)PIPE_REPLICATED) ZeROOptimizer)version) get_global_norm_of_tensorsclip_tensors_by_global_norm DummyOptimalign_dense_tensorsall_gather_dp_groupsis_model_parallel_parametersee_memory_usage graph_processget_norm_with_moe_layers)link_hp_params#lazy_init_hp_params_optimizer_statefragment_addressgroups) is_moe_paramis_moe_param_group)bwc_tensor_model_parallel_rank)register_grad_hook)enable_universal_checkpoint) DS_VERSIONPARTITION_COUNTBASE_OPTIMIZER_STATESINGLE_PARTITION_OF_FP32_GROUPS CLIP_GRADGROUP_PADDINGSPARAM_SLICE_MAPPINGSrFcCs(tdkr|s |rt|dSdSdSNr)distget_rankprint)messagedebugforcer)T/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/bf16_optimizer.py print_rank_0s r+cseZdZ          dIfdd Zd d Zd d Zd dZddZddZddZ ddZ ddZ ddZ ddZ edJddZdKdd Zed!d"ZedLd#d$ZedLd%d&Zed'd(ZedLd)d*Zed+d,Zd-d.ZdLd/d0ZdMd1d2Zd3d4Zd5d6Zd7d8Z     dNd9d:ZdOd;d<Zd=d>Z d?d@Z!e"dAdBZ#e"dCdDZ$dEdFZ%dGdHZ&Z'S)PBF16_OptimizerNr TFTc sFttddd||_||_||_t|jt |_| t j t j fvs+Jd| | |_ | |_ ||_||_||_t||_|_tj|jd|_| |_g|_fddtt|jjD|_|jrg|t|_t |_!d|_"g|_#g|_$g|_%g|_&g|_'i|_(g|_)g|_*g|_+g|_,g|_-| |_.|jr|/td dddS) Nzbegin bf16_optimizerTr(zFsz+BF16_Optimizer.__init__..r.zend bf16_ optimizer)0super__init__rtimers optimizer param_names isinstancer using_real_optimizertorchfloat32bfloat16grad_acc_dtypeimmediate_grad_update clip_grad norm_typempuintallgather_bucket_sizer6r#r$dp_rankhas_moe_layersnon_expert_gradientsrangelen param_groupsreal_dp_process_group_configure_moe_settingsrflattenr unflattennccl_start_alignment_factor bf16_groupsbf16_groups_flatbf16_partitioned_groupsfp32_groups_flat_partitionfp32_groups_gradientsfp32_groups_gradient_dictfp32_groups_gradients_flat!fp32_groups_actual_gradients_flat#fp32_groups_gradient_flat_partitionfp32_groups_has_gradientsgroup_paddingsgraph_harvesting_setup_for_real_optimizer) selfinit_optimizerr<rFrDrErHr6r:rBr_rCrJ __class__r5r*r9%sP   zBF16_Optimizer.__init__cCsXt|jjD]\}}|j|D] }t|ddrd|_qq|jD]}|qtddS)N _hp_mappingzRemoved grad acc hooks) enumerater;rNrTgetattrre_grad_acc_hooksremover+)rar4_phookr)r)r*destroygs    zBF16_Optimizer.destroycCstdd|jjDsJdt|jjD]!\}}t|r6tdd|dDs,Jdt|d|j|<qi|_ |j rKt D] }g|j |<qCdSdS)NcSg|]}t|qSr))r)r3r2r)r)r*r7rz:BF16_Optimizer._configure_moe_settings..zThe model has moe layers, but None of the param groups are marked as MoE. Create a param group with 'moe' key set to True before creating optimizercSrnr))rr3paramr)r)r*r7ws paramsz*All params in MoE group must be MoE paramsname) anyr;rNrfrallr_get_expert_data_parallel_grouprOexpert_gradientsrJ$_get_expert_data_parallel_group_dictkeys)rar4r2keyr)r)r*rPps(  z&BF16_Optimizer._configure_moe_settingsc sddjD_tjjD]&\}tjjd}tdddtjjd}dd|dD}j |j  j j |jj j d j |fd dt|D}j |j ||dj_d dj D}tjj jd }j |jrt|rj|d  |nj |jj|d}j ||j!<t"|} j# t$jdd| j% t$jd|j& dgt'j |tjjddkrj | } nd} j( | jg|d<tdddqg_)j*rB+,d_-./_0dS)NcSsg|]}tj|dqS)r1)r#get_world_size)r3pgr)r)r*r7sz.r1zbefore initializing group Tr0cSsg|]}|jr|qSr)) requires_gradrpr)r)r*r7srr) tensor_list flat_tensorcs$g|]}jd|qS)r)rUnarrow)r3dp_indexr4partition_sizerar)r*r7scSsg|]}|qSr))numel)r3tr)r)r*r7ro)dtypers)r num_elem_listrFzafter initializing group )1rOpartition_countrfr;rNr#r{rr$rTappendrU_flatten_dense_tensors_alignedrS#_update_storage_to_flattened_tensorrrLrVrWclonefloatdetachr}r? zeros_likerBrZrJrrwrK_split_flat_tensorrXrYsumr[rr\r]rMr^rhrCcreate_grad_acc_hooks_link_all_hp_params_hp_optimizer_states_linked_enable_universal_checkpoint_create_param_mapping_param_slice_mappings) ra param_groupreal_dp_world_size partition_idtrainable_parametersbf16_dp_partitionsrfp32_flat_bufferfp32_gradientslength_without_paddingpaddingr)rr*r`sl           z(BF16_Optimizer._setup_for_real_optimizercCs|jD]}t|dqdS)N) param_list)rTr)ralp_param_groupr)r)r*rs  z+BF16_Optimizer._enable_universal_checkpointcCs`g}t|jjD]%\}}t}|j|D]}|jdur'|j|}|j||<q||q|SN) rfr;rNrrTrer<get_hp_fragment_addressr)ra param_mappingr4rjparam_mapping_per_grouplplp_namer)r)r*rs   z$BF16_Optimizer._create_param_mappingc Cst|jjD]:\}}tj|j|d}tj|j|d}|j||}|j |}t |j |||j dd|||||j|d qdS)Nr1F) lp_param_listflat_hp_partition gradient_dictoffload_gradient_dict use_offloadparam_group_indexpartition_startrdp_group) rfr;rNr#r{rOr$rUrrWrrTrY)rar4rjrrrrr)r)r*rs   z"BF16_Optimizer._link_all_hp_paramscCsF|js!t|jjD]\}}t|j||j||jjq d|_dSdS)NT)rrfr;rNrrTrWstate)rar4rjr)r)r*$_lazy_init_hp_params_optimizer_states z3BF16_Optimizer._lazy_init_hp_params_optimizer_statecCsLt||ks Jg}d}|D]}t|d||}||||7}q|Sr")rrr?rr)rarrr~offsetnum_elem dense_tensorr)r)r*rs  z!BF16_Optimizer._split_flat_tensorcCs,|||}t||D]\}}|j|_q dSr)rRzipdata)rar~rupdated_paramsrkqr)r)r*rs  z2BF16_Optimizer._update_storage_to_flattened_tensorcCs|t||Sr)rQr )rar~ alignmentr)r)r*r z-BF16_Optimizer._flatten_dense_tensors_alignedcCs|dur t|jd|\}}t||j|j|jd}|}|jr,t||j||jd}||_ |dks5J|j dkrJt |jdd|j ||j|jdt |j |jD]\}}|j|jkra||jn||_qQ|j|jtjury|j D]}d|_qs|||dS)Nz does not support closure.) input_tensorsrFrE use_graph)rFexpert_tensorsrEr-T) for_clipping)rmax_norm global_normrFr)NotImplementedErrorrdget_grads_for_normr rFrEr_rJr_global_grad_normrDr rrWr\rtogradr;steprBr?r@rupdate_lp_paramsclear_hp_grads)raclosurenon_expert_grads_for_normexpert_grads_for_normnon_expert_groups_normall_groups_normparam_partitiongrad_partitionr)r)r*rsP         zBF16_Optimizer.stepcKs4||jdd|i||r|j|ddSdS)aePerform a backward pass and copy the low-precision gradients to the high-precision copy. We copy/accumulate to the high-precision grads now to prevent accumulating in the bf16 grads after successive backward() calls (i.e., grad accumulation steps > 1) The low-precision grads are deallocated during this procedure. retain_graphclear_lp_gradsNr))rbackwardupdate_hp_grads)ralossrrr bwd_kwargsr)r)r*r=s  zBF16_Optimizer.backwardcCs|jdurdS|j||}|dus"Jdt|d|d|d|j|jj|j|j||_ d|j ||<|rE|j dSdS)Nz4high precision param has no gradient, lp param_id = z group_info = [z][]T) rrXidradd_rrviewshape_hp_gradr]zero_)rar group_idx param_idxrhp_gradr)r)r*_update_hp_gradLs   zBF16_Optimizer._update_hp_gradcCs:t|jD]\}}t|D] \}}|||||q qdSr)rfrTrrarr4r2jrr)r)r*_update_hp_grads_func]s z$BF16_Optimizer._update_hp_grads_funccCsn|jrdS|jrtd|j|n||t|jD]\}}t|D]\}}|jdur,q"d|j||<q"qdS)NFT)rCr_rrrfrTrr]rr)r)r*rcs  zBF16_Optimizer.update_hp_gradscCs|jr |j|jfS|jifSr)rJrKrwrar)r)r*get_grads_for_reductionss  z&BF16_Optimizer.get_grads_for_reductionc Cs&i}g}g}t|jd}t|jt|jjksJt|jD]l\}}t|D]c\}} |sBt| tr5| j r5q&|dksBt | sBt | sBq&|j ||sJq&|s|jj|} |j rtt| rt| d|vreg|| d<|| d|j||q&||j||q&||j||q&q|s||fS|S)z Returns: tuple[list[Tensor], dict[ep_name, List[Tensor]] | list: If for_clipping, return all gradients. Otherwise, separate and return dict of expert_grad and list of non_expert_grad )rFrrs)rrFrMrTr;rNrfhasattrrds_pipe_replicatedrrr]rJrrrX) rarrrall_grads_for_cliptensor_mp_rankr4r2rrrr)r)r*rys4     z!BF16_Optimizer.get_grads_for_normcCsdtt|j|jD]\}\}}tj|j|d}||j|jq t |j |j|j|j |j ddS)Nr1) groups_flatpartitioned_param_groupsr6start_alignment_factorrH) rfrrVrWr#r$rOrcopy_r rUrSrHrar4bf16_partitionsfp32_partitionrr)r)r*rs  zBF16_Optimizer.update_lp_paramscCs@|jD]}|qt|jD]\}}dgt||j|<qdS)NF)rZrrfrXrMr])raflat_gradientsr4r2r)r)r*rs  zBF16_Optimizer.clear_hp_gradscCs|jr |r Jdg}|jD]#}|D]}|rd|_q|jdur0|jjdur*|j||jqq|sAt|dkrCt|dSdSdS)Nz>graph harvesting is incompatible with setting lp grads to Noner) r_rTrgrad_fndetach_rrMr?_foreach_zero_)ra set_to_nonezero_grads_listr2rqr)r)r*rs       zBF16_Optimizer.clear_lp_gradscCs|||dSr)rr)rarr)r)r* zero_grads  zBF16_Optimizer.zero_gradcCsPi}|j|t<|j|t<|j|t<|j|t<|j |t <t |t <|j |t<|Sr)rDrr; state_dictrrWrr^r rrrrrr!)rarr)r)r*rs     zBF16_Optimizer.state_dictcCsHtt|j|jD]\}\}}tj|j|d}|j||jq dS)Nr1) rfrrVrWr#r$rOrrrr)r)r*_restore_from_bit16_weightss  z*BF16_Optimizer._restore_from_bit16_weightscCs |dSr)rrr)r)r*refresh_fp32_paramss z"BF16_Optimizer.refresh_fp32_paramscCs(|r ||||dS||||dSr)_load_universal_checkpoint_load_legacy_checkpoint)rastate_dict_listcheckpoint_folderload_optimizer_statesload_from_fp32_weights load_serial param_shapesr)r)r*load_state_dictszBF16_Optimizer.load_state_dictc Cstj|jd}||}|td}|sJdt|}|t|j|_|r2t d|j |t |rOt |j|tD]\}}t||} |j| jq<|rW|dSdS)Nr1Fz8Empty ds_version in checkpoint, not clear how to proceedz=_load_legacy_checkpoint current_rank_sd[BASE_OPTIMIZER_STATE])r#r$r6getr pkg_versionparserrDr%r;rrrrWr_get_padded_tensorrrrr) rarrrrIcurrent_rank_sd ckpt_versioncurrentsaved src_tensorr)r)r*rs$     z&BF16_Optimizer._load_legacy_checkpointcCs|d|dS)NrT),load_hp_checkpoint_state_from_checkpoint_dir)rarrrr)r)r*r rz)BF16_Optimizer._load_universal_checkpointcCsdSrr))rasdr)r)r*_load_global_statesz!BF16_Optimizer._load_global_statecC|jjS)z+Forward the wrapped optimizer's parameters.)r;rNrr)r)r*rNzBF16_Optimizer.param_groupscCr)z'Forward the wrapped optimizer's states.)r;rrr)r)r*rrzBF16_Optimizer.statecCs |jsJ|j|||dddS)NFr)rCr)ralp_paramrrr)r)r*!accumulate_hp_grads_and_remove_lps z0BF16_Optimizer.accumulate_hp_grads_and_remove_lpcsHtjD]\}}t|D]\}}|jr fdd}||||q qdS)Ncs(fdd}jt|dS)NcsdSr)r) notneeded)r4rrqrar)r*r&sz`BF16_Optimizer.create_grad_acc_hooks..wrapper..accumulate_hp_grads_and_remove_lp)rhrr)rqr4rrr)r4rrqr*wrapper$sz5BF16_Optimizer.create_grad_acc_hooks..wrapper)rfrTr})rar4rrrqrr)rr*rs  z$BF16_Optimizer.create_grad_acc_hooks) Nr-r.r/NNNFTFr)FTF)F)T)NTFNN)TF)(__name__ __module__ __qualname__r9rmrPr`rrrrrrrr?no_gradrrrrrrrrrrrrrrrrrrpropertyrNrrr __classcell__r)r)rcr*r,#stB T    -     -       r,cCsJ||kr|Stj||j|jd}t|dd|}|j|j|S)N)rdevicer)rr?zerosrrrrr)rsize padded_tensor slice_tensorr)r)r*r .s r )FF)8 collectionsrr?sys torch._utilsrr deepspeedrr#deepspeed.runtime.constantsr deepspeed.runtime.base_optimizerr packagingrrdeepspeed.git_version_infodeepspeed.runtime.utilsr r r r r rrrrdeepspeed.utilsrrrrdeepspeed.moe.utilsrrdeepspeed.utils.bwcrdeepspeed.utils.torchrdeepspeed.checkpointrdeepspeed.checkpoint.constantsrrrrrr r!setattrmodulesrr+r,r r)r)r)r*s0      ,   $