o
    ߥi	                     @   sl   d Z ddlZddlmZ dddZdddZG dd	 d	ejjZG d
d dejjZ	dddZ
dddZdS )z-
Copyright 2020 The Microsoft DeepSpeed Team
    N)mpuc                    s^       t } fddtt D } ||< tjj| t d tj	||d  }|S )z5Gather tensors and concatenate them along a dimensionc                    s   g | ]}t  qS  )torch
empty_like).0_input_r   ^/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/gpt_moe/moe/mappings.py
<listcomp>   s    z"_gather_tokens.<locals>.<listcomp>)group)dim)

contiguousr   get_tensor_model_parallel_rankrange$get_tensor_model_parallel_world_sizer   distributed
all_gatherget_tensor_model_parallel_groupcat)r	   r   ranktensor_listoutputr   r   r
   _gather_tokens	   s   


r   c                 C   sf   t  }t  }| j| | dks"J d| d| j|  d| d| j| | }t| ||| |S )z/Divide a tensor among the tensor parallel ranksr   zinput dimension z (z2) is not divisible by tensor parallel world size ())r   r   r   shaper   narrow)r	   r   total_chunks
this_chunk
chunk_sizer   r   r
   _drop_tokens   s   r    c                   @   4   e Zd ZdZedd Zedd Zedd ZdS )	_GatherTokensz1All gather tokens among the tensor parallel ranksc                 C   
   t ||S N)r   graphr	   r   r   r   r
   symbolic-      
z_GatherTokens.symbolicc                 C      || _ t||S r$   )r   r   ctxr	   r   r   r   r
   forward1      
z_GatherTokens.forwardc                 C      t || jd fS r$   )r    r   )r+   grad_outputr   r   r
   backward6      z_GatherTokens.backwardN__name__
__module____qualname____doc__staticmethodr'   r,   r0   r   r   r   r
   r"   *       

r"   c                   @   r!   )	_DropTokensz5Divide tokens equally among the tensor parallel ranksc                 C   r#   r$   )r    r%   r   r   r
   r'   >   r(   z_DropTokens.symbolicc                 C   r)   r$   )r   r    r*   r   r   r
   r,   B   r-   z_DropTokens.forwardc                 C   r.   r$   )r   r   )r+   r	   r   r   r
   r0   G   r1   z_DropTokens.backwardNr2   r   r   r   r
   r9   ;   r8   r9   c                 C   $   t d u s
t  dkr| S t| |S N   )r   r   r"   applyr	   r   r   r   r
   gather_tokensL      r?   c                 C   r:   r;   )r   r   r9   r=   r>   r   r   r
   drop_tokensS   r@   rA   )r   )r6   r   megatron_utilr   r   r    autogradFunctionr"   r9   r?   rA   r   r   r   r
   <module>   s    


