o
    }o™i®  ã                   @   s>   d dl Z d dlmZ d dlmZ d dlmZ defdd„ZdS )é    N)ÚOptional)ÚHyenaConfig)ÚFLOPSConfigÚconfigc                 C   s|  t ƒ }|j}|j}|j}dtt fdd„}|| jƒ\}}}}	d| j | j | j	 | j
 }
d| j | j | j	d  }d| j | j | j	d  }d| j | j | j | j	 }d| j | j	 | jd  }d| j | j | | j	 }d| j | j | | j	 }d| j | j | | j	 }| jd | j t | j¡ | j	 }d|
| j|| |   |	|  || | |  ||  ||  ||   S )	z6Model FLOPs for Hyena family. FPL = 'flops per layer'.Úmodel_patternc                 S   st   d\}}}}| du rdS | D ]%}|dkr|d7 }q|dkr"|d7 }q|dkr+|d7 }q|dkr3|d7 }q||||fS )z‰Count how many small, medium, and large Hyena layers there are in the model. Also, count the
        number of Attention layers.
        )r   r   r   r   NÚSé   ÚDÚHÚ*© )r   r   r	   r
   ÚAÚlayerr   r   úS/home/ubuntu/.local/lib/python3.10/site-packages/nemo/utils/hyena_flops_formulas.pyÚ_hyena_layer_count%   s   


€z!hyena.<locals>._hyena_layer_counté   é   é   é
   é   )r   Úshort_conv_LÚhyena_short_conv_lenÚhyena_medium_conv_lenr   Ústrr   ÚgbsÚenc_seq_lenÚhsÚ
vocab_sizeÚffn_hsÚmathÚlog2Úlayers)r   Úhyena_configÚhyena_short_conv_Lr   r   r   r   r	   r
   r   Ú
logits_fplÚpre_attn_qkv_proj_fplÚpost_attn_proj_fplÚglu_ffn_fplÚattn_fplÚhyena_proj_fplÚhyena_short_conv_fplÚhyena_medium_conv_fplÚhyena_long_conv_fft_fplr   r   r   Úhyena   s<   "ÿþýüûúÿr-   )r   Útypingr   Ú:nemo.collections.llm.gpt.model.megatron.hyena.hyena_configr   Únemo.utils.flops_formulasr   r-   r   r   r   r   Ú<module>   s
   