o
    }oi                     @   s6   d dl m  mZ d dlmZmZ dd Zdd ZdS )    N)StarcoderConfigStarcoderConfig15Bc                  C   s   t dddd} | jdksJ | jtjksJ | jdu sJ | jdks$J | jdks+J | jd	ks2J | j	d	ks9J | j
d
ks@J | jdksGJ | jdu sNJ | jdksUJ | jdks\J | jdu scJ | jdu sjJ | jdu sqJ d S )N(   0      )
num_layersnum_attention_headshidden_size	LayerNormTi    learned_absoluteg?g{Gz?gh㈵>F      )r   normalizationactivation_funcFgeluadd_bias_linear
seq_lengthposition_embedding_typehidden_dropoutattention_dropoutinit_method_stdlayernorm_epsilon#share_embeddings_and_output_weightskv_channelsnum_query_groupsattention_softmax_in_fp32bias_activation_fusionbias_dropout_fusionconfig r!   b/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/gpt/model/test_starcoder.pytest_starcoder_config   s    r#   c                  C   sP   t  } | jdks
J | jdksJ | jdksJ | jdksJ | jdks&J d S )Nr   r   i `  r   g{Gz?)r   r   r	   ffn_hidden_sizer   r   r   r!   r!   r"   test_starcoder_config_15b'   s   r%   )	torch.nn.functionalnn
functionalr   (nemo.collections.llm.gpt.model.starcoderr   r   r#   r%   r!   r!   r!   r"   <module>   s   