o
    }oi\                     @   sB   d dl m  mZ d dlmZmZmZ dd Zdd Z	dd Z
dS )	    N)MistralConfig7BMistralNeMoConfig12BMistralNeMoConfig123Bc                  C   s   t  } | jdks
J | jtjksJ | jdksJ | jdu s J | jdu s'J | jdks.J | j	dks5J | j
dks<J | jdksCJ | jdksJJ | jd	ksQJ | jd
ksXJ | jd
ks_J | jdu sfJ | jdksmJ | jdkstJ | jddgks}J d S )NRMSNormropeFT           8  i           {Gz?h㈵>r   )r   normalizationactivation_funcFsiluposition_embedding_typeadd_bias_lineargated_linear_unit
num_layershidden_sizenum_attention_headsnum_query_groupsffn_hidden_size
seq_lengthattention_dropouthidden_dropout#share_embeddings_and_output_weightsinit_method_stdlayernorm_epsilonwindow_sizeconfig r#   `/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/gpt/model/test_mistral.pytest_mistral_config7b   s$   r%   c                  C   $  t  } | jdks
J | jtjksJ | jdksJ | jdu s J | jdu s'J | jdks.J | j	dks5J | j
dks<J | jdksCJ | jd	ksJJ | jd
ksQJ | jdksXJ | jdks_J | jdu sfJ | jdksmJ | jdkstJ | jd u s{J | jdksJ | jdksJ | jdksJ d S )Nr   r   FT(   i   r   r	   r
   r   r   r   r         ?    .A   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    rotary_percentrotary_basekv_channelsr!   r#   r#   r$   test_mistral_nemo_config_12b)   *   r.   c                  C   r&   )Nr   r   FTX   i 0  `   r	   i p  r   r   r   r   r(   r)   r*   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r+   r,   r-   r!   r#   r#   r$   test_mistral_nemo_config_123bA   r/   r2   )torch.nn.functionalnn
functionalr   &nemo.collections.llm.gpt.model.mistralr   r   r   r%   r.   r2   r#   r#   r#   r$   <module>   s
   