o
    i                     @   s   d dl Z d dlmZ d dlmZmZmZ dd Z					dddZ						 			dd
dZ				dddZ
G dd de jjZG dd de jjZdS )    N)init)DropoutAddLayerNormFn%DropoutAddLayerNormParallelResidualFnDropoutAddLayerNormSubsetFnc                 C   s   t | d |d d d d|dddS )N        FTr   apply)xweightepsilon r   M/home/ubuntu/vllm_env/lib/python3.10/site-packages/flash_attn/ops/rms_norm.pyrms_norm   s   r   Fc                 C   s    t | ||||||||	|d|
S zmresidual_in_fp32 only has an effect if residual is None.
    Otherwise residual dtype is residual.dtype.
    Tr   )x0residualr
   bias	dropout_pr   rowscale
layerscaleprenormresidual_in_fp32return_dropout_maskr   r   r   dropout_add_rms_norm   s   r         ?c                 C   s&   t | |||||||||	|
||d|S r   )r   r   )r   r   r
   r   r   r   r   	x0_subset
out_subsetrowscale_constout_numrowsr   r   r   r   r   r   dropout_add_rms_norm_subset4   s"   r   c                 C   s"   t | |||||||||
|	d|S r   )r   r   )r   x1r   weight0bias0weight1bias1r   r   r   r   r   r   r   r   &dropout_add_rms_norm_parallel_residualZ   s   r%   c                       s.   e Zd Zd	 fdd	Zdd Zdd Z  ZS )
RMSNormh㈵>Nc                    sN   ||d}t    || _tjtj|fi || _| dd  | 	  d S N)devicedtyper   )
super__init__epstorchnn	Parameteremptyr
   register_parameterreset_parameters)selfhidden_sizer-   r)   r*   factory_kwargs	__class__r   r   r,   }   s   

zRMSNorm.__init__c                 C      t | j d S Nr   ones_r
   r4   r   r   r   r3         zRMSNorm.reset_parametersc                 C   s   t || j| jS r:   )r   r
   r-   )r4   r	   r   r   r   forward   r>   zRMSNorm.forward)r'   NN__name__
__module____qualname__r,   r3   r?   __classcell__r   r   r7   r   r&   |   s    r&   c                       s<   e Zd Z						d fdd	Zdd Zdd	d
Z  ZS )DropoutAddRMSNormFr   r'   Nc           	         s`   ||d}t    || _|| _|| _|| _tjtj	|fi || _
| dd  |   d S r(   )r+   r,   r   pr-   r   r.   r/   r0   r1   r
   r2   r3   )	r4   r5   r   rF   r-   r   r)   r*   r6   r7   r   r   r,      s   


zDropoutAddRMSNorm.__init__c                 C   r9   r:   r;   r=   r   r   r   r3      r>   z"DropoutAddRMSNorm.reset_parametersc              
   C   s,   t ||| jd | jr| jnd| j| j| jdS )Nr   )r   r   )r   r
   trainingrF   r-   r   r   )r4   r   r   r   r   r   r?      s   zDropoutAddRMSNorm.forward)Fr   r'   FNNr:   r@   r   r   r7   r   rE      s    rE   )NNFFF)NNNr   r   FFF)FFF)r.   torch.nnr   flash_attn.ops.layer_normr   r   r   r   r   r   r%   r/   Moduler&   rE   r   r   r   r   <module>   s2   
'
0
"