o
    ߥiN                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z
 ddlmZ ddlmZ 		dd	d
Zdd ZdddZ				dddZdd ZdS )    N)mpu)Float16Module)unwrap_model)DistributedDataParallel   )logger)MoEc              	   C   s   |du rt  }tj| |d|d}|d dkr5tj|d}tj| |d| d|dd}||fS tj|d	 }}||fS )
z8Determine the directory name for this rank's checkpoint.Nmp_rank_02dr   zmodel_rng.pt
expp_rank_	_mp_rank_z_optim_states.ptzmodel_optim_rng.ptr   get_tensor_model_parallel_rankospathjoin)checkpoints_pathpath_load_tagnum_expertstensor_rank	expp_rankcommon_path
model_name
optim_name r   _/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/gpt_moe/checkpointing.pyget_checkpoint_names   s    

r   c              
   C   s:   t  }tjtj| dd| d| d|dd}|S )Nmodellayer__expert_r   r
   z_model_states.ptr   )r   layer_id	expert_idmp_rank	ckpt_namer   r   r   _get_expert_ckpt_name4   s   r$   c           	      C   sL   t  }t |}t| |||d}|\}}td|  tj|dd}|S )z Load the base state_dict from the given directory

    If rank0 is true, just loads rank 0 checkpoint, ignoring arguments.
    )r   r   r   zLoading model checkpoint from cpumap_location)r   get_max_expert_size_nameget_expert_parallel_rankr   r   infotorchload)	load_dirr   r   largest_group_namer   checkpoint_namesmodel_checkpoint_nameoptim_checkpoint_namemodel_state_dictr   r   r   _load_base_checkpoint=   s   
r3   Tr   c                 C   s   t | ttf} t|||d}|d usJ |rt| |d | nt| |d | |r3| j|d |d n	| j|d |d tj rHtj	  d S d S )N)r   r   moduler   )strict)
r   torchDDPr   r3   load_moe_checkpointload_state_dictr+   distributedis_initializedbarrier)r   r-   r   r5   r   load_ds_ckptsr2   r   r   r   load_checkpointQ   s   
r=   c                 C   s   d}|   D ]b\}}t|trh|j}|j}t|}t|D ]E}	|| |	 }
t|||
}t	
d|  tj|tdd}d}t| D ]}|| |
 | |	 }||||< qF|| q|d7 }qd S )Nr   zLoading expert states from r%   r&   z).deepspeed_moe.experts.deepspeed_experts.r   )named_modules
isinstancer   expert_group_namenum_local_expertsr   r)   ranger$   r   r*   r+   r,   devicelistkeysreplacepopupdate)r   
state_dictr-   moe_layer_idn_moduler4   
group_namerA   r   local_expert_idglobal_expert_idmoe_load_pathexpert_state_dictmoe_str_prefixkey	local_keyr   r   r   r7   l   s4   




r7   )NN)NTr   T)r   r+   megatron_utilr   megatron_util.modelr   megatron_util.utilsr   torch.nn.parallelr   r6   configurationr   	moe.layerr   r   r$   r3   r=   r7   r   r   r   r   <module>   s&   

	
