o
    i                     @   s  d dl Z d dlmZ d dlm  mZ d dlmZ zd dlm	Z	 W n e
y,   dZ	Y nw z
d dlmZmZ W n e
yD   d\ZZY nw z
d dlmZmZ W n e
y\   d\ZZY nw G dd dejZG d	d
 d
ejZG dd dejZG dd dejZdS )    N)ProcessGroup)swiglu)ColumnParallelLinearRowParallelLinear)NN)FusedMLPParallelFusedMLPc                       s8   e Zd Zddejdddddf fdd	Zdd Z  ZS )MlpNTFc
                    s|   ||	d}
t    |d ur|n|}|d ur|n|d }|| _tj||fd|i|
| _|| _tj||fd|i|
| _d S )Ndevicedtype   bias)super__init__return_residualnnLinearfc1
activationfc2)selfin_featureshidden_featuresout_featuresr   bias1bias2r   r
   r   factory_kwargs	__class__ L/home/ubuntu/vllm_env/lib/python3.10/site-packages/flash_attn/modules/mlp.pyr      s   

zMlp.__init__c                 C   s0   |  |}| |}| |}| js|S ||fS N)r   r   r   r   r   xyr   r   r    forward/   s   


zMlp.forward)__name__
__module____qualname__Fgelur   r%   __classcell__r   r   r   r    r      s    r   c                	       s@   e Zd Zddejddddddf	def fddZdd Z  ZS )ParallelMLPNTprocess_groupc                    s   |	|
d}t    td usJ dtd usJ d|d ur |n|}|d ur(|n|d }t|||f||d|| _|| _t|||f||d|| _d S )Nr	   zNeed to install fused_denser   r   sequence_parallel)r   r   r   r   r   r   r   )r   r   r   r   r   r-   r/   r   r   r
   r   r   r   r   r    r   7   s6   

zParallelMLP.__init__c                 C   s"   |  |}| |}| |}|S r!   )r   r   r   r"   r   r   r    r%   \   s   


zParallelMLP.forward)	r&   r'   r(   r)   r*   r   r   r%   r+   r   r   r   r    r,   6   s    %r,   c                	       s:   e Zd Zddejddddddf	 fdd	Zdd Z  ZS )	GatedMlpNT   Fc                    s   |	|
d}t    |d ur|n|}|d ur|ntd| d }|| d | | }|| _tj|d| fd|i|| _|| _tj||fd|i|| _d S )Nr	               r   )	r   r   intr   r   r   r   r   r   )r   r   r   r   r   r   r   multiple_ofr   r
   r   r   r   r   r    r   d   s   

zGatedMlp.__init__c                 C   s   |  |}| jtjkrtj|dd}n)| jtjkr,td ur,|jddd\}}t||}n|jddd\}}|| | }| |}| j	sF|S ||fS N)dimr5   )
r   r   r)   sigmoidglusilur   chunkr   r   r   r#   r$   gater   r   r    r%   }   s   

zGatedMlp.forward)r&   r'   r(   r)   r;   r   r%   r+   r   r   r   r    r0   c   s    r0   c                	       s>   e Zd ZdZddejddddddf	 fdd	Zdd Z  ZS )	ParallelGatedMlpzParallel GatedMlpNTr1   c                    s   |
|d}t    |d ur|n|}|d ur|ntd| d }|| d | | }td u s2td u r6tdt|d| |f||	d|| _|| _t|||f||	d|| _d S )Nr	   r2   r3   r4   zfused_dense is not installedr5   r.   )	r   r   r6   r   r   ImportErrorr   r   r   )r   r   r-   r   r   r   r   r   r7   r/   r
   r   r   r   r   r    r      s:   

zParallelGatedMlp.__init__c                 C   sT   |  |}| jtjkrtj|dd}n|jddd\}}|| | }| |}|S r8   )r   r   r)   r;   r<   r>   r   r?   r   r   r    r%      s   

zParallelGatedMlp.forward)	r&   r'   r(   __doc__r)   r;   r   r%   r+   r   r   r   r    rA      s    )rA   )torchtorch.nnr   torch.nn.functional
functionalr)   torch.distributedr   flash_attn.ops.activationsr   rB   flash_attn.ops.fused_denser   r   r   r   Moduler   r,   r0   rA   r   r   r   r    <module>   s.   -(