o
    }oi
                     @   sZ   d dl m  mZ d dlmZmZmZmZm	Z	 dd Z
dd Zdd Zd	d
 Zdd ZdS )    N)Qwen2ConfigQwen2Config1P5BQwen2Config7BQwen2Config72BQwen2Config500Mc                  C   s   t dddd} | jdksJ | jtjksJ | jdu sJ | jdu s$J | jdu s+J | jdks2J | j	d	ks9J | j
d
ks@J | jd
ksGJ | jdksNJ | jdu sUJ | jdks\J | jdkscJ | jdksjJ d S )N        )
num_layershidden_sizenum_attention_headsRMSNormTFi   g{Gz?g        iQ gư>g    .Arope)r   normalizationactivation_funcFsilugated_linear_unitadd_bias_linearadd_qkv_bias
seq_lengthinit_method_stdhidden_dropoutattention_dropout
vocab_size#share_embeddings_and_output_weightslayernorm_epsilonrotary_baseposition_embedding_typeconfig r!   ^/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/gpt/model/test_qwen2.pytest_qwen2_config   s   r#   c                  C   P   t  } | jdks
J | jdksJ | jdksJ | jdksJ | jdks&J d S )Nr   r   r	      i   )r   r
   r   r   num_query_groupsffn_hidden_sizer   r!   r!   r"   test_qwen2_config_500m,      r(   c                  C   r$   )N   i      r%   i #  )r   r
   r   r   r&   r'   r   r!   r!   r"   test_qwen2_config_1p5b5   r)   r,   c                  C   s^   t  } | jdks
J | jdksJ | jdksJ | jdksJ | jdks&J | jdks-J d S )Nr*   i      i J   R )r   r
   r   r   r&   r'   r   r   r!   r!   r"   test_qwen2_config_7b>   s   r/   c                  C   sz   t  } | jdks
J | jdksJ | jdksJ | jdksJ | jdks&J | jdks-J | jdks4J | jdks;J d S )NP   i    @      is  r.   gh㈵>)r   r
   r   r   r&   r'   r   r   r   r!   r!   r"   test_qwen2_config_72bH   s   r3   )torch.nn.functionalnn
functionalr   $nemo.collections.llm.gpt.model.qwen2r   r   r   r   r   r#   r(   r,   r/   r3   r!   r!   r!   r"   <module>   s   			
