o
    پi                     @   s   d dl Z d dlZd dlZd dlmZmZmZ d dlmZ G dd deZ	G dd deZ
G dd	 d	eZed
kr@ejdd dS dS )    N)GemmaRMSNorm	LayerNormRMSNorm)CustomTestCasec                   @   R   e Zd ZejejgZg dZg dZddgZ	dgZ
edd Zdd	 Zd
d ZdS )TestRMSNorm   S   i   
i   i  i  i           i    i   FTr   c                 C   "   t j s
tdt d d S NzCUDA is not availablecudatorchr   is_availableunittestSkipTestset_default_devicecls r   N/home/ubuntu/.local/lib/python3.10/site-packages/sglang/test/test_layernorm.py
setUpClass      

zTestRMSNorm.setUpClassc                 C     t | t|j|d}|jjjddd dd|  }t j|||d| }|r/t || nd }	t 	  |
||	}
|||	}W d    n1 sKw   Y  |rt| t j|d |
d ddd	 | t j|d |
d ddd	 d S | t j||
ddd	 d S )
Ndtype      ?皙?meanstd      r   {Gz?atolrtol)r   manual_seedr   toweightdatanormal_randn
randn_likeinference_modeforward_native
assertTrueallcloseself
num_tokenshidden_sizeadd_residualr!   seedlayerscalexresidualref_outoutr   r   r   _run_rms_norm_test   s   

 $zTestRMSNorm._run_rms_norm_testc              	   C   x   t | j| j| j| j| jD ]+}| j|d |d |d |d |d d | j|  W d    n1 s4w   Y  qd S Nr   r'   r(         )r:   r;   r<   r!   r=   )		itertoolsproduct
NUM_TOKENSHIDDEN_SIZESADD_RESIDUALDTYPESSEEDSsubTestrD   r9   paramsr   r   r   test_rms_norm*   $   zTestRMSNorm.test_rms_normN)__name__
__module____qualname__r   halfbfloat16rN   rK   rL   rM   rO   classmethodr   rD   rS   r   r   r   r   r   
   s    
r   c                   @   r   )TestGemmaRMSNormr   r   FTr   c                 C   r   r   r   r   r   r   r   r   C   r   zTestGemmaRMSNorm.setUpClassc                 C   r   )
Nr    r"   r#   r$   r'   r(   r   MbP?r*   )r   r-   r   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r   r   r   _run_gemma_rms_norm_testI   s   

 $z)TestGemmaRMSNorm._run_gemma_rms_norm_testc              	   C   rE   rF   )	rI   rJ   rK   rL   rM   rN   rO   rP   r]   rQ   r   r   r   test_gemma_rms_norm^   rT   z$TestGemmaRMSNorm.test_gemma_rms_normN)rU   rV   rW   r   rX   rY   rN   rK   rL   rM   rO   rZ   r   r]   r^   r   r   r   r   r[   <   s    
r[   c                   @   sf   e Zd ZejejgZejejgZg dZ	g dZ
ddgZddgZdgZedd Zdd	 Zd
d ZdS )TestLayerNorm)r	   r
   i   )   i   i   r   r   r   r   i   FTr   c                 C   r   r   r   r   r   r   r   r   y   r   zTestLayerNorm.setUpClassc                 C   sl  t | t||||d}|r#|jjjddd |r#|jjjddd dd|  }	t j|||d|	 }
t   |	|
}||
}W d    n1 sKw   Y  | 
t j||d	d
d |r|r|t jkrj|t jkst j|_|jjt j|j_|jjt j|j_t   ||
t j|
j}W d    n1 sw   Y  | 
t j||dd
d d S d S d S d S )N)elementwise_affinebiasr!   r"   r#   r$   g        r'   r(   r    r)   r\   r*   g{Gz?)r   r-   r   r/   r0   r1   rb   r2   r4   r5   r6   r7   rY   float32r!   r.   )r9   r:   r;   
use_affineuse_biasr!   r=   param_dtyper>   r?   r@   rB   rC   cuda_outr   r   r   _run_layer_norm_test   s:   




z"TestLayerNorm._run_layer_norm_testc              
   C   s   t | j| j| j| j| j| j| jD ]1}| j	|d |d |d |d |d |d |d d | j
|  W d    n1 s>w   Y  qd S )	Nr   r'   r(   rG   rH         )r:   r;   rd   re   r!   r=   rf   )rI   rJ   rK   rL   
USE_AFFINEUSE_BIASrN   rO   PARAM_DTYPESrP   rh   rQ   r   r   r   test_layer_norm   s,   		zTestLayerNorm.test_layer_normN)rU   rV   rW   r   rX   rY   rN   rc   rm   rK   rL   rk   rl   rO   rZ   r   rh   rn   r   r   r   r   r_   p   s    
#r_   __main__r(   )	verbosity)rI   r   r   sglang.srt.layers.layernormr   r   r   sglang.test.test_utilsr   r   r[   r_   rU   mainr   r   r   r   <module>   s    24H