o
    }oi                     @   s>   d dl Zd dlZd dlZd dlmZmZmZ G dd dZdS )    N)openai_gelu
quick_gelusquared_reluc                   @   sd   e Zd ZdZejdd Zejdd Zdd Zdd	 Z	d
d Z
dd Zdd Zdd Zdd ZdS )TestActivationFunctionszJTest suite for activation functions in nemo.collections.llm.fn.activation.c                 C   s   t jg dt jdS )z;Returns a tensor with test values for activation functions.)g       g      g                    ?      ?       @dtype)torchtensorfloat32self r   \/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/fn/test_activation.pyinput_tensor   s   z$TestActivationFunctions.input_tensorc                 C   s   t jddt jdS )z9Returns a larger tensor for testing activation functions.   
   r
   )r   randnr   r   r   r   r   larger_input_tensor   s   z+TestActivationFunctions.larger_input_tensorc                 C      t |}tjg dtjd}tj||ddsJ t||dk t||dk s+J t||dk dks8J t||dk  dk sEJ dS )z)Test the openai_gelu activation function.)g9>g{$SĿgAÿr   g`- ?gv?g16:F?r
   h㈵>atolr   N)r   r   r   r   allclose
zeros_likeallr   r   resultexpectedr   r   r   test_openai_gelu#      
&z(TestActivationFunctions.test_openai_geluc                 C   r   )z(Test the quick_gelu activation function.)gx':xgR`ÿguڦx&ÿr   gƒl?g+gJ?gQ,u?r
   r   r   r   N)r   r   r   r   r   r   r   r   r   r   r   test_quick_gelu4   r#   z'TestActivationFunctions.test_quick_geluc                 C   s4   t |}tjg dtjd}tj||ddsJ dS )z*Test the squared_relu activation function.)r   r   r   r   g      ?r   g      @r
   r   r   N)r   r   r   r   r   r   r   r   r   test_squared_reluE   s   z)TestActivationFunctions.test_squared_reluc                 C   sL   t |}t|}t|}|j|jksJ |j|jksJ |j|jks$J dS )z?Test that all activation functions preserve input tensor shape.N)r   r   r   shape)r   r   gelu_outputquick_gelu_outputsquared_relu_outputr   r   r   test_activation_shapesN   s   z.TestActivationFunctions.test_activation_shapesc                 C   s>   t ddd}dd }||}t|}t j||ddsJ dS )	zFTest that openai_gelu is close to the mathematical definition of GELU.   d   c                 S   s6   d|  dt tdtj | dt | d     S )Nr         gHm?r,   )r   tanhnpsqrtpipow)xr   r   r   
paper_gelu]   s   6zPTestActivationFunctions.test_gelu_implementation_equivalence.<locals>.paper_gelur   r   N)r   linspacer   r   )r   r5   r6   r!   r    r   r   r   $test_gelu_implementation_equivalenceX   s
   z<TestActivationFunctions.test_gelu_implementation_equivalencec                 C   s@   t jdgt jd}t|}t jdgt jd}t ||sJ dS )z!Test that squared_relu of 0 is 0.r   r
   N)r   r   r   r   r   )r   r5   r    r!   r   r   r   test_squared_relu_0e   s   z+TestActivationFunctions.test_squared_relu_0c                 C   s  t  u t jg dt jdd}t|}| }|  |jdur+t |jt 	|r-J |j
  t|}| }|  |jdurMt |jt 	|rOJ |j
  t|}| }|  |jdurot |jt 	|rqJ W d   dS 1 s|w   Y  dS )zDTest that gradients flow correctly through the activation functions.)r   r   r	   T)r   requires_gradN)r   enable_gradr   r   r   sumbackwardgradr   r   zero_r   r   )r   r5   y_gelu	loss_geluy_quick
loss_quick	y_squaredloss_squaredr   r   r   test_gradient_flowm   s"   
"
"
$"z*TestActivationFunctions.test_gradient_flowN)__name__
__module____qualname____doc__pytestfixturer   r   r"   r$   r%   r*   r8   r9   rF   r   r   r   r   r      s    

	
r   )	numpyr1   rK   r   "nemo.collections.llm.fn.activationr   r   r   r   r   r   r   r   <module>   s
   