o
     ÇÏiú  ã                   @   s&  d dl Z ddlmZmZmZmZmZmZmZm	Z	m
Z
mZmZmZmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZ dd	lmZmZmZmZ dd
l m!Z!m"Z"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m,Z, ddl-m.Z.m-Z- ddl/m0Z0m1Z1m/Z/ eZ2ddd„Z3g d¢Z4dS )é    Né   )ÚAttentionBiasÚAttentionOpÚAttentionOpBaseÚLowerTriangularMaskÚMemoryEfficientAttentionCkOpÚ+MemoryEfficientAttentionCutlassFwdFlashBwOpÚ!MemoryEfficientAttentionCutlassOpÚ(MemoryEfficientAttentionFlashAttentionOpÚ"MemoryEfficientAttentionSplitKCkOpÚmemory_efficient_attentionÚ#memory_efficient_attention_backwardÚ"memory_efficient_attention_forwardÚ0memory_efficient_attention_forward_requires_grad)Úindex_select_catÚscaled_index_add)ÚColumnParallelLinearÚRowParallelLinear)ÚRMSNorm)Úrope_padded)Ú sequence_parallel_leading_matmulÚ!sequence_parallel_trailing_matmul)Úfused_allgather_and_anythingÚfused_allgather_and_linearÚ fused_anything_and_reducescatterÚfused_linear_and_reducescatter)ÚSparse24TensorÚ
sparsify24Úsparsify24_like)ÚSwiGLUÚSwiGLUEagerOpÚSwiGLUFusedOpÚSwiGLUOpÚSwiGLUOpDispatchÚSwiGLUPackedFusedOpÚswiglu)Útiled_matmul)ÚTreeAttnMetadataÚtree_attention)Úget_stack_stridesÚstack_or_noneÚunbindc                 C   sŽ   t j | ||f¡rt j t| ||f| ||¡S | | }|d u r |S |jt jkrA|jdkr8| d¡ 	|j
d dd¡}tdƒ|| < |S ||7 }|S )Né   r   éÿÿÿÿz-inf)ÚtorchÚ	overridesÚhas_torch_functionÚhandle_torch_functionÚmasked_matmulÚdtypeÚboolÚndimÚ	unsqueezeÚexpandÚshapeÚfloat)ÚaÚbÚmaskÚatt© r>   úI/home/ubuntu/.local/lib/python3.10/site-packages/xformers/ops/__init__.pyr2   5   s   ÿ
ÿr2   )+r   ÚAttentionMaskr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r   r   r   r2   )N)5r.   Úfmhar   r   r   r   r   r   r	   r
   r   r   r   r   r   Úindexingr   r   Úmodpar_layersr   r   Úrmsnormr   r   Úseqparr   r   Úsequence_parallel_fused_opsr   r   r   r   Úsp24r   r   r   Ú	swiglu_opr   r    r!   r"   r#   r$   r%   r&   r(   r'   r+   r)   r*   r@   r2   Ú__all__r>   r>   r>   r?   Ú<module>   s    <$	
