o
    Ti&                     @   s   d dl Z d dlmZ 	 da			ddedededed	ed
e jfddZ	 			ddedededed	ed
e jfddZG dd de j	j
ZG dd de j	j
ZdS )    N)RandomLTDBuilder   cpureserved_length
seq_length
batch_sizelayersdevice	attn_maskc           	      C   s   t j|| |f|d}t || }|||| t j}td u r%t  at	||}|d urC|d d d d d | d | f }||fS d }||fS )Nr	   )
torchonesmultinomialreshapetoint32random_ltd_moduler   loadtoken_sort_)	r   r   r   r   r	   r
   	prob_distsampled_indicesnew_mask r   [/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/ops/random_ltd/dropping_utils.pygpt_sample_tokens   s   
 r   c              
   C   s  |d usJ t j|| |f|d}t || }|||| t j}td u r+t  at	||}|j
}|t j}g }	t|D ]>}
g }t|D ]+}|||d d d ||
 | d d f }||d d d d d d ||
 | f  qH|	t j|dd q@|||	fS )Nr   r   r   )dim)r   r   r   r   r   r   r   r   r   r   dtypelongrangeappendcat)r   r   r   r   r	   r
   r   r   r   r   ltmp_mask_listimask_tmpr   r   r   bert_sample_tokens4   s"   
(,r%   c                   @   sD   e Zd ZedejdejdefddZedejdejfdd	Zd
S )GatherTokensactivationssorted_indicesbatch_firstc                 C   s6   t d u r	t  a | || || _|t |||fS N)r   r   r   save_for_backwardr)   token_gather)ctxr'   r(   r)   r   r   r   forwardT   s
   
zGatherTokens.forwarda_gradientsg_gradientsc                 C   s@   |  }td u rt  a| j\}}| j}t||||d d fS r*   )
contiguousr   r   r   saved_tensorsr)   token_scatter_)r-   r/   r0   r'   r(   r)   r   r   r   backward]   s   

zGatherTokens.backwardN	__name__
__module____qualname__staticmethodr   Tensorboolr.   r4   r   r   r   r   r&   R   s
    r&   c                	   @   sD   e Zd ZedejdejdejdefddZedejfdd	Zd
S )ScatterTokensall_activationslayer_activationsr(   r)   c                 C   s:   t d u r	t  a t | |||}| | || _|S r*   )r   r   r   r3   cloner+   r)   )r-   r=   r>   r(   r)   scatter_resultsr   r   r   r.   l   s   

zScatterTokens.forwardout_gradientsc                 C   sB   |  }td u rt  a| j\}| j}t|||}||d d fS r*   )r1   r   r   r   r2   r)   r,   )r-   rA   r(   r)   ret_valr   r   r   r4   y   s   
zScatterTokens.backwardNr5   r   r   r   r   r<   j   s    r<   )r   r   N)r   deepspeed.ops.op_builderr   r   intstrr:   r   r%   autogradFunctionr&   r<   r   r   r   r   <module>   sF   


