o
    پi"                     @   sH  d Z ddlZddlmZ ddlZddlmZ ddlm  mZ	 ddl
mZ e Ze Ze Zes5er;ddlmZ erAddlZddlmZ edG dd	 d	eZed
G dd deZedG dd deZedG dd deZejedd ejejedZde dej!fddZ"eedZ#de dej!fddZ$dS )zCustom activation functions.    N)Any)current_platform)silu_and_mul)CustomOpr   c                       sd   e Zd ZdZd fddZdejdejfddZdejdejfd	d
ZdejdejfddZ	  Z
S )
SiluAndMulzAn activation function for SwiGLU.

    The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2.

    Shapes:
        x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
        return: (num_tokens, d) or (batch_size, seq_len, d)
    returnNc                       t    d S Nsuper__init__self	__class__ c/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/runtime/layers/activation.pyr   '      zSiluAndMul.__init__xc                 C   sD   |j d d }|j d d |f }tj||j|jd}t|| |S )N   )dtypedevice)shapetorchemptyr   r   r   )r   r   doutput_shapeoutr   r   r   forward_cuda*   s
   
zSiluAndMul.forward_cudac                 C   s4   |j d d }t|dd|f |d|df  S )6PyTorch-native implementation equivalent to forward().r   r   .N)r   Fsilur   r   r   r   r   r   forward_native1   s   &zSiluAndMul.forward_nativec                 C   s   t |}|S r	   )	torch_npu
npu_swiglu)r   r   r   r   r   r   forward_npu6   s   
zSiluAndMul.forward_npu)r   N)__name__
__module____qualname____doc__r   r   Tensorr   r$   r'   __classcell__r   r   r   r   r      s    	r   gelu_and_mulc                       sZ   e Zd ZdZddef fddZdefddZd	ej	dej	fd
dZ
defddZ  ZS )
GeluAndMulzAn activation function for GeGLU.

    The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2.

    Shapes:
        x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d)
        return: (batch_size, seq_len, d) or (num_tokens, d)
    noneapproximatec                    s*   t    || _|dvrtd| d S )N)r0   tanhzUnknown approximate mode: )r   r   r1   
ValueError)r   r1   r   r   r   r   F   s
   
zGeluAndMul.__init__r   c                 O      | j |i |S r	   r$   r   argskwargsr   r   r   r   L      zGeluAndMul.forward_cudar   c                 C   s:   |j d d }tj|dd|f | jd|d|df  S )r    r   r   .Nr1   )r   r!   gelur1   r#   r   r   r   r$   O   s   ,zGeluAndMul.forward_nativec                 C   s   dt | j S )Nzapproximate=)reprr1   r   r   r   r   
extra_reprT   r9   zGeluAndMul.extra_repr)r0   )r(   r)   r*   r+   strr   r   r   r   r,   r$   r=   r-   r   r   r   r   r/   ;   s    	r/   gelu_newc                       @   e Zd Z fddZdefddZdejdejfddZ  Z	S )	NewGELUc                    r   r	   r
   r   r   r   r   r   [   r   zNewGELU.__init__r   c                 O   r4   r	   r5   r6   r   r   r   r   ^   r9   zNewGELU.forward_cudar   c                 C   s:   t dt j }d| dt||dt|d     S )r    g       @g      ?g      ?gHm?g      @)mathsqrtpir   r2   pow)r   r   cr   r   r   r$   a   s   *zNewGELU.forward_native
r(   r)   r*   r   r   r   r   r,   r$   r-   r   r   r   r   rA   X       rA   
quick_geluc                       r@   )		QuickGELUc                    r   r	   r
   r   r   r   r   r   j   r   zQuickGELU.__init__r   c                 O   r4   r	   r5   r6   r   r   r   r   m   r9   zQuickGELU.forward_cudar   c                 C   s   |t d|  S )r    gZd;?)r   sigmoid)r   r   r   r   r   r$   p   s   zQuickGELU.forward_nativerG   r   r   r   r   rJ   g   rH   rJ   c                   C   s   t jddS )Nr2   r:   )nnGELUr   r   r   r   <lambda>x   s    rN   )r;   r?   gelu_pytorch_tanhrelur"   rI   act_fn_namer   c                 C   *   |   } | tvrtd| dt|   S )z#Get an activation function by name.Activation function  is not supported.)lower_ACTIVATION_REGISTRYr3   rQ   r   r   r   
get_act_fn      
rX   )r;   r"   c                 C   rR   )z=Get an activation-and-mul (i.e. SiluAndMul) function by name.rS   rT   )rU   _ACTIVATION_AND_MUL_REGISTRYr3   rW   r   r   r   get_act_and_mul_fn   rY   r[   )%r+   rB   typingr   r   torch.nnrL   torch.nn.functional
functionalr!   'sglang.multimodal_gen.runtime.platformsr   is_cuda_is_cudais_hip_is_hipis_npu_is_npu
sgl_kernelr   r%   .sglang.multimodal_gen.runtime.layers.custom_opr   registerr   r/   rA   rJ   rM   ReLUSiLUrV   r>   ModulerX   rZ   r[   r   r   r   r   <module>   sF   

