o
     i5                     @   s  d dl Z d dlmZ d dlmZmZmZmZmZm	Z	m
Z
 d dlZddlmZmZmZmZmZmZ ddlmZmZmZ e
de	e e	e Zdad	ed
dfddZd
efddZd
efddZded
efddZdeded
dfddZdee d
efddZ 	d&dedee dedeeeeee f   d
ef
ddZ!deded
ee	e  fdd Z"deded
e	e fd!d"Z#ded#ee d
e	e fd$d%Z$dS )'    N)deque)AnyListOptionalSequenceTupleTypeTypeVar   )	attn_biasckcutlassflashflash3triton_splitk)AttentionBwOpBaseAttentionFwOpBaseInputsTTuse_flash_attention3returnc                 C   s   | a d S N_USE_FLASH_ATTENTION_3)r    r   N/home/ubuntu/.local/lib/python3.10/site-packages/xformers/ops/fmha/dispatch.py_set_use_fa3   s   r   c                   C   s   t S r   r   r   r   r   r   _get_use_fa3   s   r   c                  C   s0   t jjd u} | ot j dk}tjd u}|o|S )N)	   r   )torchversioncudaget_device_capabilityr   _C_flashattention3)has_cudais_90ahas_valid_flash3r   r   r   fa3_available    s   
r'   inpc                 C   s`   dt | jj d| jj dt | jj d| jj dt | jj d| jj dt| j d| j S )Nzquery       : shape=z (z)
key         : shape=z)
value       : shape=z)
attn_bias   : z
p           : )	tuplequeryshapedtypekeyvaluetyper   p)r(   r   r   r   _format_inputs_description'   s   

r1   namec              	   C   s>   | |}|s	d S | d| dtt|d dt|| )Nz
Operator `z` does not support inputs:
     
)not_supported_reasonstextwrapindentr1   _format_not_supported_reasons)exc_typer2   opr(   reasonsr   r   r   _ensure_op_supports_or_raise/   s   
r<   r;   c                 C   s   d| j  dd| S )N`z ` is not supported because:
    z
    )NAMEjoin)r:   r;   r   r   r   r8   :   s   r8   priority_listextra_op_reasonsc                 C   s   g }|D ]}| |}|s|  S || qd|  dtt|d }t||D ]\}}|dt|| 7 }q*|d urL|D ]\}}|dt|| 7 }q>t|)NzNo operator found for `z` with inputs:
r3   r4   )r5   appendr6   r7   r1   zipr8   NotImplementedError)r2   r@   r(   rA   r5   r:   not_supportedmsgr   r   r   _run_priority_list>   s   
rG   needs_gradientc                 C   sX  t jjrt rtjgng }t|tjtjg }ntt	jg}|s| j
jdko5| j
ddko5| j
jd dk}|r| jjd dkr| j
jd dkrd}| jjdkrW| jjd }n%| jjdkrj| jjd | jjd  }n| jjd	kr|| jjd | jjd  }|dkr|d
k r|tj t jjrt| jtjst r|tj |tj |tj |S )N   r   r
                   @   )r   r    r!   r   r   FwOpr   r   r   r   r-   ndimstrider+   r*   
appendleftr   
isinstancer   BlockDiagonalMaskremove)r(   rH   	flash3_oppriority_list_ops
mqa_or_gqaparallelism_BHr   r   r   _dispatch_fw_priority_listV   sD   *$
r\   c                 C   s   t dt| || S )zComputes the best operator for forward

    Raises:
        NotImplementedError: if not operator was found

    Returns:
        AttentionOp: The best operator for the configuration
    "memory_efficient_attention_forward)rG   r\   )r(   rH   r   r   r   _dispatch_fw   s
   	r^   varlen_lse_packedc                    s   t jjrtjtjg}t rtjg| }ntjg}g }t	| j
t
jrT| j
jjjd dkrT d us1J |D ]}|j krJ||d rBdnd dgf q3 fdd|D }td	|| S )
Nr   rN   z
LSE is in packedpaddedz formatc                    s   g | ]	}|j  kr|qS r   )VARLEN_LSE_PACKED).0r:   r_   r   r   
<listcomp>   s    z _dispatch_bw.<locals>.<listcomp>#memory_efficient_attention_backward)r   r    r!   r   BwOpr   r   r   r   rU   r   VARLEN_BIASES	q_seqinfoseqstartr+   rb   rB   rG   )r(   r_   rY   rA   r:   r   rd   r   _dispatch_bw   s:   

rk   r   )%r6   collectionsr   typingr   r   r   r   r   r   r	   r    r   r   r   r   r   r   commonr   r   r   r   r   boolr   r   r'   strr1   r<   r8   rG   r\   r^   rk   r   r   r   r   <module>   sR   $ 


/