o
    ߥi                     @   s,   d Z ddlZddlZG dd dejjZdS )z-
Copyright 2020 The Microsoft DeepSpeed Team
    Nc                       s&   e Zd Zd fdd	Zdd Z  ZS )Experts   Nc                    sb   t t|   tj fddt|D | _|| _| jD ]  	 D ]
\}}d|_
||_q#qd S )Nc                    s   g | ]}t  qS  )copydeepcopy).0iexpertr   ]/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/gpt_moe/moe/experts.py
<listcomp>   s    z$Experts.__init__.<locals>.<listcomp>F)superr   __init__torchnn
ModuleListrangedeepspeed_expertsnum_local_expertsnamed_parameters	allreduce
group_name)selfr
   r   expert_group_namenameparam	__class__r	   r   r      s   
zExperts.__init__c                 C   sb   |j | jdd}g }t|| jD ]\}}||}t|tu r"|d }||g7 }qtj|dd}|S )Nr   )dimr   )chunkr   zipr   typetupler   cat)r   inputschunksexpert_outputsr   r
   outexpert_outputr   r   r   forward   s   zExperts.forward)r   N)__name__
__module____qualname__r   r)   __classcell__r   r   r   r   r   
   s    r   )__doc__r   r   r   Moduler   r   r   r   r   <module>   s    