o
    Ni|                     @   s   d dl mZ d dlZd dlmZmZmZmZ d dlm  mZ	 d dl
mZmZ d dlmZmZmZmZmZ d dlmZ dd Zd	d
 ZG dd deZG dd deZdS )    )
namedtupleN)nntensorpi	is_tensor)Module
ModuleList)	rearrangerepeateinsumpackunpack)Encoderc                 C   s   | d uS N )vr   r   J/home/ubuntu/.local/lib/python3.10/site-packages/x_transformers/neo_mlp.pyexists   s   r   c                 C   s   t | r| S |S r   )r   )r   dr   r   r   default   s   r   c                       s$   e Zd Z fddZdd Z  ZS )RandomFourierEmbedc                    s(   t    td|| _| jd d S )N   F)super__init__r   Linearprojrequires_grad_)selfdim	__class__r   r   r      s   
zRandomFourierEmbed.__init__c                 C   s&   t |d}| |}tdt | S )Nz... -> ... 1   )r	   r   torchcosr   )r   times	rand_projr   r   r   forward   s   

zRandomFourierEmbed.forward)__name__
__module____qualname__r   r&   __classcell__r   r   r   r   r      s    r   c                       sB   e Zd ZdZ	 edddddef fddZ		dd
dZ  ZS )NeoMLPz, https://openreview.net/forum?id=A8Vuf2e8y6       )attn_dim_headheads)encoder_kwargsr0   c                   s   t    tt||| _tt||| _tt||| _tj	j
| jdd tj	j
| jdd tj	j
| jdd tt|t||| _td||d|| _tt||| _tt|| _d S )Ng{Gz?)std)r   depthr   )r   r   r   	Parameterr"   zerosinput_embedhidden_embedoutput_embedinitnormal_
Sequentialr   r   random_fourierr   transformerrandnto_output_weightsto_output_bias)r   dim_in
dim_hiddendim_out	dim_modelr2   r0   r   r   r   r   .   s&   

zNeoMLP.__init__Fc                    s   |j dk}|rt|d}|jd  | |}|| j }t fdd| j| jfD \}}t|||gd\}}	| 	|}t
||	d\}}}t|| jd}
|
| j }
|rWt|
d}
|s[|
S |
|||ffS )	Nr   z... -> 1 ...r   c                 3   s    | ]
}t |d  dV  qdS )z... -> b ...)bN)r
   ).0tbatchr   r   	<genexpr>n   s    z!NeoMLP.forward.<locals>.<genexpr>zb * dzb n d, n d -> b nz1 ... -> ...)ndimr	   shaper;   r5   tupler6   r7   r   r<   r   r   r>   r?   )r   xreturn_embedsno_batchfouriered_inputr5   r6   r7   embedpacked_shapeoutputr   rG   r   r&   \   s"   




"


zNeoMLP.forward)F)r'   r(   r)   __doc__dictr   r&   r*   r   r   r   r   r+   *   s    
1r+   )collectionsr   r"   r   r   r   r   torch.nn.functional
functionalFtorch.nnr   r   einopsr	   r
   r   r   r   x_transformers.x_transformersr   r   r   r   r+   r   r   r   r   <module>   s    