o
    ¾e¦iË  ã                   @   s2   d dl mZ d dlmZ G dd„ deƒZdgZdS )é   )ÚPreTrainedConfig)ÚRopeParametersc                ,       s:  e Zd ZdZdZdgZdddddddœZdgdgfd	d
gd	gfd	gd	gfdœZ																				d1dedB dedB dedB dedB dedB d edB d!e	dB d"edB d#e
dB d$e
dB d%edB d&edB d'edB d(edB d)edB d*edB d+e
dB d,edB d-edB d.eee	ef B dB f(‡ fd/d0„Z‡  ZS )2ÚJais2ConfigaÕ
  
    This is the configuration class to store the configuration of a [`Jais2Model`]. It is used to instantiate a Jais2
    model according to the specified arguments, defining the model architecture.
    [inceptionai/Jais-2-8B-Chat](https://huggingface.co/inceptionai/Jais-2-8B-Chat).

    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 150272):
            Vocabulary size of the Jais2 model.
        hidden_size (`int`, *optional*, defaults to 3328):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 26624):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 26):
            Number of attention heads for each attention layer.
        num_key_value_heads (`int`, *optional*):
            Number of key_value heads for Grouped Query Attention.
        hidden_act (`str`, *optional*, defaults to `"relu2"`):
            The non-linear activation function in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer.
        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether to return last key/values attentions.
        pad_token_id (`int`, *optional*):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 0):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 150024):
            End of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings.
        attention_bias (`bool`, *optional*, defaults to `True`):
            Whether to use a bias in the query, key, value and output projection layers.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        mlp_bias (`bool`, *optional*, defaults to `True`):
            Whether to use a bias in up_proj, down_proj and gate_proj layers.
        head_dim (`int`, *optional*):
            The attention head dimension.
        rope_parameters (`dict`, *optional*):
            The RoPE parameters.
    Újais2Úpast_key_valuesÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormé K é   é h  é    é   NÚrelu2é    ç{®Gáz”?çñhãˆµøä>Té    éJ Fç        Ú
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚ
hidden_actÚmax_position_embeddingsÚinitializer_rangeÚlayer_norm_epsÚ	use_cacheÚpad_token_idÚbos_token_idÚeos_token_idÚtie_word_embeddingsÚattention_biasÚattention_dropoutÚmlp_biasÚhead_dimÚrope_parametersc                    s®   || _ || _|| _|| _|| _|| _|d u r|}|| _|| _|	| _|| _	|| _
|| _|| _|d ur3|n| j| j | _|| _|| _|| _|| _|| _tƒ jdi |¤Ž |
| _d S )N© )r   r#   r   r   r   r    r!   r"   r$   r&   r+   r,   r-   r.   r/   r*   r'   r(   r)   ÚsuperÚ__init__r%   )Úselfr   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   Úkwargs©Ú	__class__r0   úk/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/jais2/configuration_jais2.pyr2   _   s.   
zJais2Config.__init__)r   r   r   r   r   Nr   r   r   r   TNr   r   FTr   TNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚintÚstrÚfloatÚboolr   Údictr2   Ú__classcell__r0   r0   r5   r7   r      s˜    3ú
	
ýëþýüûúùø	÷
öõôóòñðïîíìër   N)Úconfiguration_utilsr   Úmodeling_rope_utilsr   r   Ú__all__r0   r0   r0   r7   Ú<module>   s   
z