o
    }oi	                     @   sT   d dl mZ d dlmZmZmZmZmZ dd Zdd Z	dd Z
d	d
 Zdd ZdS )    )openai_gelu)CodeGemmaConfig2BCodeGemmaConfig7BGemmaConfigGemmaConfig2BGemmaConfig7Bc                  C   s   t dd} | jdksJ | jtksJ | jdu sJ | jdks!J | jdu s(J | jdks/J | jdks6J | j	d	ks=J | j
d	ksDJ | jdu sKJ | jdu sRJ d S )
N   )
num_layersRMSNormTropeFi       g        )r   normalizationactivation_funcr   gated_linear_unitposition_embedding_typeadd_bias_linear
seq_lengthkv_channelsattention_dropouthidden_dropout#share_embeddings_and_output_weightslayernorm_zero_centered_gammaconfig r   ^/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/gpt/model/test_gemma.pytest_gemma_config   s   
r   c                  C   P   t  } | jdks
J | jdksJ | jdksJ | jdksJ | jdks&J d S Nr   i         i @  )r   r	   hidden_sizenum_attention_headsnum_query_groupsffn_hidden_sizer   r   r   r   test_gemma_config_2b(      r%   c                  C   P   t  } | jdks
J | jdksJ | jdksJ | jdksJ | jdks&J d S N   i      i `  )r   r	   r!   r"   r#   r$   r   r   r   r   test_gemma_config_7b1   r&   r+   c                  C   r   r   )r   r	   r!   r"   r#   r$   r   r   r   r   test_code_gemma_config_2b:   r&   r,   c                  C   r'   r(   )r   r	   r!   r"   r#   r$   r   r   r   r   test_code_gemma_config_7bC   r&   r-   N)"nemo.collections.llm.fn.activationr   $nemo.collections.llm.gpt.model.gemmar   r   r   r   r   r   r%   r+   r,   r-   r   r   r   r   <module>   s   				