o
    ei4                     @   sT   d dl mZ d dlmZ ddlmZmZ G dd deZG dd deZddgZ	d	S )
   )PreTrainedConfig)RopeParameters   )CONFIG_MAPPING
AutoConfigc                /       sP  e Zd ZdZdZdgZddddddddZdgdgfd	d
gd	gfd	gd	gfdZdZ																								d4de	dB de	dB de	de	dB de	dB d e	dB d!e
dB d"e	dB d#edB d$e	dB d%edB d&e	dB d'e	dB d(e	dB d)edB d*eee
ef B dB d+edB d,edB d-edB d.e	dB d/e	d0e	d1e	f. fd2d3Z  ZS )5AriaTextConfigau  
    This class handles the configuration for the text component of the Aria model.
    Instantiating a configuration with the defaults will yield a similar configuration to that of the model of the Aria
    [rhymes-ai/Aria](https://huggingface.co/rhymes-ai/Aria) architecture.
    This class extends the LlamaConfig to include additional parameters specific to the Mixture of Experts (MoE) architecture.

    Args:
        vocab_size (`int`, *optional*, defaults to 32000):
            Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`LlamaModel`]
        hidden_size (`int`, *optional*, defaults to 4096):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 4096):
            The size of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 32):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details, check out [this
            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
            `num_attention_heads`.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 2048):
            The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens,
            Llama 2 up to 4096, CodeLlama up to 16384.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 2):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 1):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 2):
            End of stream token id.
        pretraining_tp (`int`, *optional*, defaults to 1):
            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
            document](https://huggingface.co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism) to
            understand more about it. This value is necessary to ensure exact reproducibility of the pretraining
            results. Please refer to [this issue](https://github.com/pytorch/pytorch/issues/76232).
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
        rope_parameters (`RopeParameters`, *optional*):
            Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
            a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE
            with longer `max_position_embeddings`.
        attention_bias (`bool`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        mlp_bias (`bool`, *optional*, defaults to `False`):
            Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.
        head_dim (`int`, *optional*):
            The attention head dimension. If None, it will default to hidden_size // num_heads
        moe_num_experts (`int`, *optional*, defaults to 8):
            The number of experts in the MoE layer.
        moe_topk (`int`, *optional*, defaults to 2):
            The number of top experts to route to for each token.
        moe_num_shared_experts (`int`, *optional*, defaults to 2):
            The number of shared experts.
    	aria_textpast_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormtext_config }         Nsilu   {Gz?ư>Tr      F           
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_heads
hidden_actmax_position_embeddingsinitializer_rangerms_norm_eps	use_cachebos_token_ideos_token_idpretraining_tptie_word_embeddingsrope_parametersattention_biasattention_dropoutmlp_biashead_dimmoe_num_expertsmoe_topkmoe_num_shared_expertsc                    s   || _ || _|| _|| _|| _|| _|| _|| _ || _|| _|d u r$|}|| _	|| _
|	| _|
| _|| _|| _|| _|| _|| _|d urE|n| j| j | _|| _|| _|| _|| _|| _t jdi | d S )N )r    r2   r3   r4   r   r%   r   r!   r"   r#   r$   r&   r'   r+   r(   r.   r/   r0   r1   r-   r,   pad_token_idr)   r*   super__init__)selfr   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r6   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   kwargs	__class__r5   i/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/aria/configuration_aria.pyr8   s   s8   zAriaTextConfig.__init__)r   r   r   r   r   Nr   r   r   r   Tr   r   r   r   FNFr   FNr   r   r   )__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planbase_config_keyintstrfloatboolr   dictr8   __classcell__r5   r5   r;   r=   r      s    G


	
r   c                       sp   e Zd ZdZdZddiZeedZ								
dde	dede
dB de	dB dedB dedB f fddZ  ZS )
AriaConfiga  
    This class handles the configuration for both vision and text components of the Aria model,
    as well as additional parameters for image token handling and projector mapping.
    Instantiating a configuration with the defaults will yield a similar configuration to that of the model of the Aria
    [rhymes-ai/Aria](https://huggingface.co/rhymes-ai/Aria) architecture.

    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information.

    Args:
        vision_config (`AriaVisionConfig` or `dict`, *optional*):
            Configuration for the vision component.
        vision_feature_layer (`int`, *optional*, defaults to -1):
            The index of the layer to select the vision feature.
        text_config (`AriaTextConfig` or `dict`, *optional*):
            Configuration for the text component.
        projector_patch_to_query_dict (`dict`, *optional*):
            Mapping of patch sizes to query dimensions.
        image_token_index (`int`, *optional*, defaults to 9):
            Index used to represent image tokens.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated normal initializer for initializing all weight matrices.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
    ariaimage_token_idimage_token_index)r   vision_configN	   r   Fvision_feature_layerr   projector_patch_to_query_dictr&   r,   c           	         s   || _ |d u rddd}dd | D | _t| j | _|| _t|tr6d|d< t	|d  di |}n	|d u r?t	d  }|| _
|| _t|trVd|v rVtdi |}n|d u r]t }|| _|| _t jdi | d S )	N      )i  i$  c                 S   s   i | ]\}}t |t |qS r5   )rG   ).0kvr5   r5   r=   
<dictcomp>   s    z'AriaConfig.__init__.<locals>.<dictcomp>idefics3_visionrB   r5   )rP   itemsrU   maxvalues'max_value_projector_patch_to_query_dictrT   
isinstancerK   r   rQ   r&   r   r   r,   r7   r8   )	r9   rQ   rT   r   rU   rP   r&   r,   r:   r;   r5   r=   r8      s,   

zAriaConfig.__init__)NrR   NNrS   r   F)r>   r?   r@   rA   rB   attribute_mapr   r   sub_configsrG   rK   rI   rJ   r8   rL   r5   r5   r;   r=   rM      s4    
rM   N)
configuration_utilsr   modeling_rope_utilsr   autor   r   r   rM   __all__r5   r5   r5   r=   <module>   s    L