o
    Ti#                     @   sB   d dl Z d dlmZmZ d dlZd dlmZ G dd dejZdS )    N)ListOptional)nnc                	       sL   e Zd Zddejdedee ddf fddZd	e	j
de	j
fd
dZ  ZS )Experts   Nexpertnum_local_expertsexpert_group_namereturnc                    s\   t t|   t fddt|D | _|| _| jD ]   D ]}d|_	||_
q"qd S )Nc                    s   g | ]}t  qS  )copydeepcopy).0_r   r   I/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/moe/experts.py
<listcomp>   s    z$Experts.__init__.<locals>.<listcomp>F)superr   __init__r   
ModuleListrangedeepspeed_expertsr   
parameters	allreduce
group_name)selfr   r   r	   param	__class__r   r   r      s   
zExperts.__init__inputsc                 C   s\   |j | jdd}g }t|| jD ]\}}||}t|tr!|d }||g7 }qtj|ddS )Nr   )dimr   )chunkr   zipr   
isinstancetupletorchcat)r   r   chunksexpert_outputsr!   r   outr   r   r   forward   s   
zExperts.forward)r   N)__name__
__module____qualname__r   Moduleintr   strr   r%   Tensorr*   __classcell__r   r   r   r   r      s    &r   )r   typingr   r   r%   r   r.   r   r   r   r   r   <module>   s
   