o
    Ti                     @   s\   d dl Z d dlZd dlmZ d dlmZ dadd Zdd Z	G dd	 d	e j
jZd
d ZdS )    N)EvoformerAttnBuilder)get_acceleratorc           
   	   C   s  | j d dksJ dtj| | jd}t | sJ dt |s'J dt |s1J dt |s;J dt |sEJ d	td u rNt  a| j d
 }| j d d d d }t	
| j d d }tj|||ftj| jd}	t| ||||||	 ||	fS )N   zseq_len must be greater than 16dtypeQ must be on cudaK must be on cudaV must be on cudazbias1 must be on cudazbias2 must be on cuda       r   device)shapetorch
empty_liker   r   on_acceleratorkernel_r   loadnpprodemptyfloat32r   	attention)
QKVbias1bias2Onheadsnqnblse r%   b/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/ops/deepspeed4science/evoformer_attn.py
_attention   s   

r'   c
                 C   sh  t |jd |jd dksJ dtj||jd}
tj||jd}tj||jd}t | s3J dt |s=J dt |sGJ dt |sQJ dt |s[J d	td u rdt 	 at|}|rttj
|tjd}n
tjg tj|jd
}|	rtj
|tjd}n
tjg tj|jd
}t| |||||||||
|||| |
|||| j|| jfS )N@   z?Hidden size is too large. Need to change kMax to a larger valuer   zdO must be on cudar   r	   r
   zO must be on cudar   )maxr   r   r   r   r   r   r   r   r   
zeros_liker   tensorr   attention_bwdto)dOr   r   r   r    r$   r   r   
bias1_grad
bias2_graddQdKdVdeltadB1dB2r%   r%   r&   r-   !   s(   "

$r-   c                   @   s&   e Zd ZedddZedd ZdS )EvoformerFusedAttentionNc           
   	   C   s   |dur|  n	tjg |j|jd}|dur|  n	tjg |j|jd}|  }|  }|  }t|||||\}}	| |||||	|| |S )z4
        q, k, v: are in shape [*, L, H, D]
        Nr   )
contiguousr   r,   r   r   r'   save_for_backward)
ctxqkvr   r   bias1_bias2_or$   r%   r%   r&   forward=   s   $$zEvoformerFusedAttention.forwardc                 C   s   | j \}}}}}}}| dko| jd }	| dko| jd }
t|||||||||	|

\}}}}}|	s6d }|
s:d }|||||fS )Nr         )saved_tensorsnumelneeds_input_gradr-   )r;   grad_outputr<   r=   r>   rA   r$   r   r   
is_b1_grad
is_b2_gradr2   r3   r4   r6   r7   r%   r%   r&   backwardK   s   $z EvoformerFusedAttention.backward)NN)__name__
__module____qualname__staticmethodrB   rK   r%   r%   r%   r&   r8   ;   s
    r8   c                 C   s   t |dksJ t |dkr|d  t |dkr|d  dd }dd }|d d ur9|d j|| ks9J d|d d urL|d j|| ksLJ dt| |||d |d S )	N   r      c                 S   s    | j d | j d dd| j d fS )Nr   rQ   rP   r   xr%   r%   r&   <lambda>a   s     z+DS4Sci_EvoformerAttention.<locals>.<lambda>c                 S   s&   | j d d| j d | j d | j d fS )Nr   rQ   rC   rP   rR   rS   r%   r%   r&   rU   b   s   & zbias1 shape is incorrectzbias2 shape is incorrect)lenappendr   r8   apply)r   r   r   biasesbias_1_shapebias_2_shaper%   r%   r&   DS4Sci_EvoformerAttentionX   s   

r\   )r   numpyr   deepspeed.ops.op_builderr   deepspeed.acceleratorr   r   r'   r-   autogradFunctionr8   r\   r%   r%   r%   r&   <module>   s   