o
    ei8                     @   sj  d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZ ddlmZ e e!Z"G dd de	Z#G dd deZ$G dd deZ%G dd deZ&G dd deZ'G dd deZ(G dd deZ)G dd deZ*G dd  d eZ+G d!d" d"eZ,g d#Z-dS )$    )CallableN)nn   )ACT2CLS)Cache)PreTrainedConfig)RopeParameters)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargslogging   )
LlamaAttentionLlamaDecoderLayerLlamaForCausalLMLlamaForTokenClassification
LlamaModelLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbeddingapply_rotary_pos_embeager_attention_forward)NemotronMLPc                &       s,  e Zd ZdZdZdgZdZdddddddZdgd	gfd
dgd
gfd
gd
gfdZdddddddddddddddddddddd dd!fd"e	dB d#e	dB d$e	dB d%e	dB d&e	dB d'e	dB d(e
dB d)e	dB d*edB d+edB d,edB d-e	dB d.e	dB d/e	dB d0edB d1edB d2edB d3edB f$ fd4d5Z  ZS )6ApertusConfiga  
    This is the configuration class to store the configuration of a [`ApertusModel`]. It is used to instantiate a Apertus
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the Apertus-8B.
    e.g. [swiss-ai/Apertus-8B](https://huggingface.co/swiss-ai/Apertus-8B)

    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information.


    Args:
        vocab_size (`int`, *optional*, defaults to 131072):
            Vocabulary size of the Apertus model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`ApertusModel`]
        hidden_size (`int`, *optional*, defaults to 4096):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 14336):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 32):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details, check out [this
            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
            `num_attention_heads`.
        hidden_act (`str` or `function`, *optional*, defaults to `"xielu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 65536):
            The maximum sequence length that this model might ever be used with. Apertus supports up to 65536 tokens.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 3):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 1):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 2):
            End of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
        rope_parameters (`RopeParameters`, *optional*):
            Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
            a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE
            with longer `max_position_embeddings`.
        attention_bias (`bool`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.

    ```python
    >>> from transformers import ApertusModel, ApertusConfig

    >>> # Initializing a Apertus-8B style configuration
    >>> configuration = ApertusConfig()

    >>> # Initializing a model from the Apertus-8B style configuration
    >>> model = ApertusModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```apertuspast_key_valuesg    `fAcolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormi   i   i 8      Nxielui   g{Gz?gh㈵>Tr      r   Fllama3g       @i    g      ?g      @)	rope_type
rope_thetafactor original_max_position_embeddingslow_freq_factorhigh_freq_factor        
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_heads
hidden_actmax_position_embeddingsinitializer_rangerms_norm_eps	use_cachepad_token_idbos_token_ideos_token_idtie_word_embeddingsrope_parametersattention_biasattention_dropoutc                    s   || _ || _|| _|| _|| _|| _|d u r|}|| _|| _|	| _|
| _	|| _
|| _|| _|| _|| _|| _|| _|| _t jdi | d S )N )r0   r7   r1   r2   r3   r4   r5   r6   r8   r9   r:   r@   rA   r?   r>   r;   r<   r=   super__init__)selfr0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   kwargs	__class__rB   i/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/apertus/modular_apertus.pyrD      s*   zApertusConfig.__init__)__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencedefault_thetabase_model_tp_planbase_model_pp_planintstrfloatboolr   rD   __classcell__rB   rB   rG   rI   r   -   s    G
	
	
r   c                          e Zd Z fddZ  ZS )
ApertusMLPc                    s\   t  | tj| j| jdd| _tj| j| jdd| _|jdkr,t	d |j
d| _d S d S )NF)biasr&   )dtype)rC   rD   r   Linearr1   r2   up_proj	down_projr6   r   r[   act_fn)rE   configrG   rB   rI   rD      s   
zApertusMLP.__init__)rJ   rK   rL   rD   rW   rB   rB   rG   rI   rY          rY   c                   @      e Zd ZdS )ApertusRMSNormNrJ   rK   rL   rB   rB   rB   rI   rc          rc   c                   @   rb   )ApertusRotaryEmbeddingNrd   rB   rB   rB   rI   rf      re   rf   c                       s   e Zd ZddededB f fddZ		ddejdeejejf dejdB d	e	dB d
ej
dB dee deejejf fddZ  ZS )ApertusAttentionNr`   	layer_idxc                    s2   t  || t| j|j| _t| j|j| _d S N)rC   rD   rc   head_dimr9   q_normk_normrE   r`   rh   rG   rB   rI   rD      s   zApertusAttention.__init__r    position_embeddingsr!   r   cache_positionrF   returnc                 K   s,  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}| |	}	| |
}
|\}}t	|	|
||\}	}
|d ura|||d}|
|
|| j|\}
}t| jjt}|| |	|
||f| jsudn| j| jd|\}}|jg |dR   }| |}||fS )Nr'   r   )sincosro   r/   )dropoutscaling)shaperj   q_projview	transposek_projv_projrk   rl   r   updaterh   r	   get_interfacer`   _attn_implementationr   trainingrA   ru   reshape
contiguouso_proj)rE   r    rn   r!   r   ro   rF   input_shapehidden_shapequery_states
key_statesvalue_statesrs   rr   cache_kwargsattention_interfaceattn_outputattn_weightsrB   rB   rI   forward   s<   	



zApertusAttention.forwardri   )NN)rJ   rK   rL   r   rS   rD   torchTensortupler   
LongTensorr
   r   r   rW   rB   rB   rG   rI   rg      s&    
rg   c                       s   e Zd Zdedef fddZ						ddejdejdB d	ejdB d
e	dB de
dB dejdB deejejf dB dee deej fddZ  ZS )ApertusDecoderLayerr`   rh   c                    s>   t  || t|j|jd| _t|j|jd| _| `| `d S )N)eps)	rC   rD   rc   r1   r9   attention_layernormfeedforward_layernorminput_layernormpost_attention_layernormrm   rG   rB   rI   rD     s
   zApertusDecoderLayer.__init__NFr    r!   position_idsr   r:   ro   rn   rF   rp   c              
   K   s^   |}	|  |}| jd|||||||d|\}}
|	| }|}	| |}| |}|	| }|S )N)r    r!   r   r   r:   ro   rn   rB   )r   	self_attnr   mlp)rE   r    r!   r   r   r:   ro   rn   rF   residual_rB   rB   rI   r     s&   




zApertusDecoderLayer.forward)NNNFNN)rJ   rK   rL   r   rS   rD   r   r   r   r   rV   r   r
   r   r   rW   rB   rB   rG   rI   r     s6    	
r   c                   @   rb   )ApertusPreTrainedModelNrd   rB   rB   rB   rI   r   ,  re   r   c                   @   rb   )ApertusModelNrd   rB   rB   rB   rI   r   0  re   r   c                       rX   )ApertusForCausalLMc                    s   t  jdi |S )an  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ApertusForCausalLM

        >>> model = ApertusForCausalLM.from_pretrained("swiss-ai/Apertus-8B")
        >>> tokenizer = AutoTokenizer.from_pretrained("swiss-ai/Apertus-8B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```NrB   )rC   r   )rE   super_kwargsrG   rB   rI   r   5  s   zApertusForCausalLM.forward)rJ   rK   rL   r   rW   rB   rB   rG   rI   r   4  ra   r   c                   @   rb   )ApertusForTokenClassificationNrd   rB   rB   rB   rI   r   O  re   r   )r   r   r   r   r   ).collections.abcr   r   r   activationsr   cache_utilsr   configuration_utilsr   modeling_rope_utilsr   modeling_utilsr	   processing_utilsr
   utilsr   r   llama.modeling_llamar   r   r   r   r   r   r   r   r   r   nemotron.modeling_nemotronr   
get_loggerrJ   loggerr   rY   rc   rf   rg   r   r   r   r   r   __all__rB   rB   rB   rI   <module>   s2   0
 	3*