o
    ̳i                     @   s*   d dl Z d dl mZ G dd dejZdS )    N)nnc                       s8   e Zd Zd
dedef fddZdd Zdd	 Z  ZS )GemmaRMSNormư>dimepsc                    s&   t    || _tt|| _d S )N)super__init__r   r   	Parametertorchzerosscale)selfr   r   	__class__ S/home/ubuntu/.local/lib/python3.10/site-packages/torchtune/models/gemma/rms_norm.pyr      s   
zGemmaRMSNorm.__init__c                 C   s$   |t |djddd| j  S )N   T)keepdim)r
   rsqrtpowmeanr   )r   xr   r   r   _norm   s   $zGemmaRMSNorm._normc                 C   s*   |  | }|d| j   }||S )Ng      ?)r   floatr   type_as)r   r   outputr   r   r   forward   s   
zGemmaRMSNorm.forward)r   )	__name__
__module____qualname__intr   r   r   r   __classcell__r   r   r   r   r      s    r   )r
   r   Moduler   r   r   r   r   <module>   s   