o
    ei]                     @   s   d dl Z d dlZd dlZddlmZ e rd dlmZ dZdZe	ej
dedZeeefvr2edi Zd	d
 Zdd Z			dddZ					dddZdS )    N   )is_torch_npu_available)npu_fusion_attention   NPU_FA2_SPARSE_MODE)defaultzEnvironment variable `NPU_FA2_SPARSE_MODE` can only be set as 2 (top-left aligned causal mask) or 3 (down-right aligned causal mask).c                 C   s4   | t vrtjtjddg| ddd t | < t |  S )z6Get or create attention mask for the specified device.i   device   )diagonal)ATTN_MASK_NPU_CACHEtorchtriuonesboolr    r   k/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/integrations/npu_flash_attention.pyget_attn_mask_npu(   s   $r   c                   C   s   t  rttkS dS )NF)r   SPARSE_MODE!TOP_LEFT_ALIGNED_CAUSAL_MASK_MODEr   r   r   r   'is_npu_fa2_top_left_aligned_causal_mask/   s   r           Fc                 K   s   d| }|d u rdt | jd  }|s(| jd }t| |||d||dd }	|	S t| j}
| jd }t| |||d|||
td	d }	|	S )N      ?r   BSND)	keep_probscaler   )r   r   
atten_masksparse_mode)mathsqrtshaper   r   r	   r   )qkv	dropout_psoftmax_scalecausalkwargsr   head_numoutputattn_mask_npur   r   r   npu_flash_attn_func3   s.   	



r,   c
                 K   s   d| }|d u rdt | jd  }|	sD| jd }t| |||d d ||dt|dd     t|dd     dd }|S t| j	}| jd }t| |||d d |||dt|dd     t|dd     t
dd }|S )Nr   r   r
   TND)pser   r   r   input_layoutactual_seq_qlenactual_seq_kvlenr   )	r.   padding_maskr   r   r   r/   r0   r1   r   )r   r    r!   r   tuplecpunumpytolistr   r	   r   )r"   r#   r$   cu_seqlens_qcu_seqlens_kmax_seqlen_qmax_seqlen_kr%   r&   r'   r(   r   r)   r*   r+   r   r   r   npu_flash_attn_varlen_funcV   sR   
 

r;   )r   NF)NNr   NF)r   osr   utils.import_utilsr   	torch_npur   r   #DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODEintgetenvr   
ValueErrorr   r   r   r,   r;   r   r   r   r   <module>   s4   
)