o
    پi                     @   sh   d dl Z d dlmZ d dlm  mZ d dl mZ d dlmZm	Z	 dde
fddZG d	d
 d
ejZdS )    N)Tensor)linear_act_funcact_linear_funcFT
activationc           	      C   s4   t | ||t ||d\}}t||||||d}|S )N)store_preactfuse_grad_accumtunedr   r   r   )r   torchis_grad_enabledr   )	xweight1weight2r   r   r   preactpostactout r   =/home/ubuntu/.local/lib/python3.10/site-packages/quack/mlp.pymlp_func
   s"   
r   c                       sL   e Zd Z									ddedef fddZd	ed
efddZ  ZS )MLPNFgeluTr   r   c                    s   ||d}t    |d ur|n|}|d ur|nd| }|| _tj||fd|i|| _tj||fd|i|| _|	| _|
| _d S )N)devicedtype   bias)	super__init__r   nnLinearfc1fc2r   r   )selfin_featureshidden_featuresout_featuresbias1bias2r   r   r   r   r   factory_kwargs	__class__r   r   r      s   


zMLP.__init__inputreturnc              	   C   s   | j jd u r?| jjd u r?|jr?|ddkr?| j jd dkr?| j jd dkr?| jjd dkr?t|| j j| jj| j	| j
| jdS |  |}| t|dd d df |ddd df  S )N      r   r	   .   )r   r   r    is_cudastrider"   r$   r   weightr   r   r   Fsilu)r!   r*   yr   r   r   forward6   s$   
	0zMLP.forward)	NNFFr   NNFT)__name__
__module____qualname__boolr   r   r6   __classcell__r   r   r(   r   r      s     
r   )FT)r
   torch.nnr   torch.nn.functional
functionalr3   r   quack.linearr   r   strr   Moduler   r   r   r   r   <module>   s   