o
    TiO                     @   s.   d dl mZ G dd dZG dd deZdS )   )SYCLOpBuilderc                   @   s   e Zd Zdd Zdd ZdS )FlashAttentionBuilderObjectc                 C      d S N selfr   r   [/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/ops/op_builder/xpu/flash_attn.py__init__
      z$FlashAttentionBuilderObject.__init__c           	      C   sD   zdd l }dd l}|jjj||||||dW S  ty!   tdw )N    )	dropout_p	is_causalscalez_Please install pytorch and intel_extension_for_pytorch to include scaled dot product attention.)torchintel_extension_for_pytorchnn
functionalscaled_dot_product_attentionImportError)	r   qkvr   softmax_scaler   r   r   r   r   r	   flash_attn_func_v2   s   
z.FlashAttentionBuilderObject.flash_attn_func_v2N)__name__
__module____qualname__r
   r   r   r   r   r	   r      s    r   c                       sV   e Zd ZdZdZd fdd	Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Z  ZS )FlashAttentionBuilderDS_BUILD_FlashAttention
flash_attnNc                    s$   |d u r| j n|}t j|d d S )N)name)NAMEsuperr
   )r   r!   	__class__r   r	   r
   !   s   zFlashAttentionBuilder.__init__c                 C   s   d| j  dS )Nzdeepspeed.ops._op)r"   r   r   r   r	   absolute_name%   s   z#FlashAttentionBuilder.absolute_namec                 C   r   r   r   r   r   r   r	   sources(   r   zFlashAttentionBuilder.sourcesc                 C      g S r   r   r   r   r   r	   include_paths+   r   z#FlashAttentionBuilder.include_pathsc                 C   r)   r   r   r   r   r   r	   extra_ldflags.   r   z#FlashAttentionBuilder.extra_ldflagsc                 C   r)   r   r   r   r   r   r	   cxx_args1   r   zFlashAttentionBuilder.cxx_argsc                 C   s   t  S r   )r   r   r   r   r	   load4   s   zFlashAttentionBuilder.loadr   )r   r   r   	BUILD_VARr"   r
   r'   r(   r*   r+   r,   r-   __classcell__r   r   r$   r	   r      s    r   N)builderr   r   r   r   r   r   r	   <module>   s   