o
     i                     @   sF   d dl Z d dlmZ d dlmZmZmZ edeG dd deZdS )    N)autocast)	AttentionAttentionConfigregister_attentionfourier_mixc                       s2   e Zd Zdef fddZdejfddZ  ZS )
FourierMixdropoutc                    s,   t    tjj|dd| _d| _d| _dS )z
        FFT-based pseudo-attention mechanism, from
        "
        "FNet: Mixing Tokens with Fourier Transforms"
        Lee-Thorp et al., 2021, https://arxiv.org/pdf/2105.03824.pdf
        F)inplaceN)super__init__torchnnDropout	attn_dropsupports_attention_maskrequires_input_projection)selfr   ___	__class__ ]/home/ubuntu/.local/lib/python3.10/site-packages/xformers/components/attention/fourier_mix.pyr      s   

zFourierMix.__init__qc                 O   sH   t ddd tj|j}W d    n1 sw   Y  | |}|S )NcudaF)enabled)r   r   fftfft2realr   )r   r   r   r   attr   r   r   forward   s
   
zFourierMix.forward)	__name__
__module____qualname__floatr   r   Tensorr    __classcell__r   r   r   r   r      s    r   )r   	torch.ampr   xformers.components.attentionr   r   r   r   r   r   r   r   <module>   s
   