o
    پiD                     @   s   d dl Z d dlZd dlZd dlmZmZ d dlmZ d dlm	Z	 e Z
G dd de	ZG dd de	Zed	kr?ejd
d dS dS )    N)
GeluAndMul	QuickGELU)is_hip)CustomTestCasec                   @   sJ   e Zd ZejejgZg dZg dZdgZ	e
dd Zdd Zdd	 Zd
S )TestGeluAndMul   S   i   i   i   i   i 6  r   c                 C   "   t j s
tdt d d S NzCUDA is not availablecudatorchr   is_availableunittestSkipTestset_default_devicecls r   O/home/ubuntu/.local/lib/python3.10/site-packages/sglang/test/test_activation.py
setUpClass      

zTestGeluAndMul.setUpClassc                 C   s   t | t j|d}t j|d| |d}t   ||}||}W d    n1 s/w   Y  |t jkr>d }	}
nd }	}
| 	t j
|||	|
d d S )Ndtype   {Gz?MbP?atolrtol)r   manual_seedr   torandninference_modeforward_nativeforward_cudabfloat16
assertTrueallclose)self
num_tokensdr   seedlayerxref_outoutr    r!   r   r   r   _run_gelu_and_mul_test   s   




z%TestGeluAndMul._run_gelu_and_mul_testc              	   C   n   t | j| j| j| jD ](}| j|d |d |d |d d | j|  W d    n1 s/w   Y  qd S )Nr      r      )r,   r-   r   r.   )	itertoolsproduct
NUM_TOKENSDDTYPESSEEDSsubTestr3   r+   paramsr   r   r   test_gelu_and_mul*   s    z TestGeluAndMul.test_gelu_and_mulN)__name__
__module____qualname__r   halfr(   r;   r9   r:   r<   classmethodr   r3   r@   r   r   r   r   r      s    
r   c                   @   s^   e Zd ZejejgZg dZg dZdgZ	e
dd Zdededejd	efd
dZdd ZdS )TestQuickGELUr   r
   r   c                 C   r   r   r   r   r   r   r   r   @   r   zTestQuickGELU.setUpClassn_tokdimr   r.   c           
      C   s   t | t j|d}t j|||dd}t   ||}tr'||}n|	|}W d    n1 s6w   Y  |t j
u rBdnd}	| jt j|||	|	dd| d| d	| d
 td| d| d	|  d S )Nr   r   )r   devicer   r   r   zMismatch @ B=z, D=z, dtype=)msgz
Match @ B=)r   r"   r   r#   r$   r%   r&   _is_hipforward_hipr'   r(   r)   r*   print)
r+   rG   rH   r   r.   r/   r0   refr2   tolr   r   r   _run_gelu_quick_testF   s    



z"TestQuickGELU._run_gelu_quick_testc              	   C   r4   )Nr   r5   r   r6   )r,   rH   r   r.   )r7   r8   r9   DIMSr;   r<   r=   rP   r>   r   r   r   test_quick_gelu[   s   zTestQuickGELU.test_quick_geluN)rA   rB   rC   r   rD   r(   r;   r9   rQ   r<   rE   r   intr   rP   rR   r   r   r   r   rF   :   s    
rF   __main__r   )	verbosity)r7   r   r   sglang.srt.layers.activationr   r   sglang.srt.utilsr   sglang.test.test_utilsr   rK   r   rF   rA   mainr   r   r   r   <module>   s    -.