o
    ॵi1                     @   s   d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d d	lmZ d d
lmZ eje	jdG dd deZdS )    N)Mapping)List)mpu)Trainers)
TorchModel)TRAINERS)NlpEpochBasedTrainer)Config)func_receive_dict_inputs)module_namec                       sH   e Zd Zdef fddZdedef fddZdd	 Zd
d Z	  Z
S )GPTMoETrainercfgc                    sJ   t  | ttjdd|j_tjdd|j_tjdd|j_	|S )N
LOCAL_RANKMASTER_ADDRz	127.0.0.1MASTER_PORT29500)
superrebuild_configintosenvirongetmodelrank	master_ipmaster_port)selfr   	__class__ [/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/nlp/gpt_moe_trainer.pyr      s
   zGPTMoETrainer.rebuild_configr   inputsc                    s.   t | }tj}t|||}t ||S N)listkeystorchint64r   broadcast_datar   
train_step)r   r   r"   r%   datatyper   r    r!   r)      s   zGPTMoETrainer.train_stepc                 C   s   | j j}|| S r#   )eval_preprocessor	tokenizer
detokenizetolist)r   tokensr,   r    r    r!   _decode"   s   zGPTMoETrainer._decodec                    s    j r jjn j}|  t   t|tr&t|j	s&|j	d	i |}n|	|}W d    n1 s5w   Y  |d } fddt
|d |D |d<  fddt
|d |D |d< t|d t|d kslJ |S )
Nprompt_lengthc                    s"   g | ]\}}  ||d  qS r#   r0   .0seqskip_lenr   r    r!   
<listcomp>3   s    z1GPTMoETrainer.evaluation_step.<locals>.<listcomp>	sequencespredsc                    s&   g | ]\}}  ||d  d qS )   Nr2   r3   r7   r    r!   r8   7   s    labelstgtsr    )_distr   moduleevalr&   no_grad
isinstancer   r
   generateziplen)r   datar   resultr1   r    r7   r!   evaluation_step&   s,   





zGPTMoETrainer.evaluation_step)__name__
__module____qualname__r	   r   r   r   r)   r0   rH   __classcell__r    r    r   r!   r      s
    r   )r   collections.abcr   typingr   r&   megatron_utilr   modelscope.metainfor   modelscope.modelsr   modelscope.trainers.builderr   modelscope.trainers.nlp_trainerr   modelscope.utils.configr	   modelscope.utils.file_utilsr
   register_modulegpt_moe_trainerr   r    r    r    r!   <module>   s   