o
    wir^                     @   s  d dl Z d dlmZmZ d dlZd dlmZ d dlZddlm	Z	m
Z
 ddlmZmZ ddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z) e*e+Z,G dd deZ-G dd de%Z.G dd de#Z/G dd de ej0Z1G dd de!Z2G dd de$Z3G dd de)Z4G dd  d e"Z5g d!Z6dS )"    N)CallableOptional   )CacheDynamicCache)PretrainedConfiglayer_type_validation)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)BaseModelOutputWithPast)rope_config_validation)ALL_ATTENTION_FUNCTIONS)Unpack)logging)deprecate_kwarg   )CohereAttentionCohereDecoderLayerCohereForCausalLMCohereLayerNormCoherePreTrainedModelCohereRotaryEmbeddingapply_rotary_pos_embeager_attention_forward)Gemma2Modelc                       s   e Zd ZdZdZdgZddddddddZdgdgfd	d
gd	gfd	gd	gfdZ																						d# fdd	Ze	d d! Z
e
jd"d! Z
  ZS )$Cohere2Configa2  
    This is the configuration class to store the configuration of a [`CohereModel`]. It is used to instantiate an Cohere
    model according to the specified arguments, defining the model architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information. Instantiating a configuration
    with the defaults will yield a similar configuration to that of the [CohereForAI/c4ai-command-r-v01](https://huggingface.co/CohereForAI/c4ai-command-r-v01) model.


    Args:
        vocab_size (`int`, *optional*, defaults to 256000):
            Vocabulary size of the Cohere model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`CohereModel`]
        hidden_size (`int`, *optional*, defaults to 8192):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 22528):
            Dimension of the MLP representations.
        logit_scale (`float`, *optional*, defaults to 0.0625):
            The scaling factor for the output logits.
        num_hidden_layers (`int`, *optional*, defaults to 40):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 64):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details, check out [this
            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
            `num_attention_heads`.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length that this model might ever be used with.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the layer normalization.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 0):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 5):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 255001):
            End of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
            Whether to tie weight embeddings
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        rope_scaling (`Dict`, *optional*):
            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
            accordingly.
            Expected contents:
                `rope_type` (`str`):
                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
                    'llama3'], with 'default' being the original RoPE implementation.
                `factor` (`float`, *optional*):
                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *
                    original maximum pre-trained length.
                `original_max_position_embeddings` (`int`, *optional*):
                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
                    pretraining.
                `attention_factor` (`float`, *optional*):
                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
                    computation. If unspecified, it defaults to value recommended by the implementation, using the
                    `factor` field to infer the suggested value.
                `beta_fast` (`float`, *optional*):
                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
                    ramp function. If unspecified, it defaults to 32.
                `beta_slow` (`float`, *optional*):
                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
                    ramp function. If unspecified, it defaults to 1.
                `short_factor` (`list[float]`, *optional*):
                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<
                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
                    size divided by the number of attention heads divided by 2
                `long_factor` (`list[float]`, *optional*):
                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<
                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
                    size divided by the number of attention heads divided by 2
                `low_freq_factor` (`float`, *optional*):
                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
                `high_freq_factor` (`float`, *optional*):
                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        sliding_window (`int`, *optional*, defaults to 4096):
            Size of the sliding window attention context.
        layer_types (`list`, *optional*):
            Attention pattern for each layer.

    ```python
    >>> from transformers import Cohere2Model, Cohere2Config

    >>> # Initializing a Cohere Nextmodel configuration
    >>> configuration = Cohere2Config()

    >>> # Initializing a model from the Cohere2 configuration
    >>> model = Cohere2Model(configuration) # doctest: +SKIP

    >>> # Accessing the model configuration
    >>> configuration = model.config # doctest: +SKIP
    ```
    cohere2past_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnorm       X        ?(   @   Nsilu{Gz?h㈵>Tr            @F           c                    s   | _ |	 _| _| _| _| _| _|d u r|}| _| _|
 _	| _
| _| _| _| _| _| _| _||  _t  t jd||||d| |dd _ jd u rst dd _ fddt jD  _t j d S )N)pad_token_idbos_token_ideos_token_idtie_word_embeddingssliding_window_pattern   c                    s&   g | ]}t |d   j rdndqS )   sliding_attentionfull_attention)bool_sliding_window_pattern).0iself h/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/cohere2/modular_cohere2.py
<listcomp>   s    z*Cohere2Config.__init__.<locals>.<listcomp>rE   )
vocab_sizemax_position_embeddingshidden_sizelogit_scaleintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_heads
hidden_actinitializer_rangelayer_norm_eps	use_cache
rope_thetarope_scalingattention_biasattention_dropoutsliding_windowlayer_typeshead_dimr   super__init__getr@   getattrranger   )rD   rH   rJ   rL   rK   rM   rN   rO   rP   rI   rQ   rR   rS   r6   r7   r8   r9   rT   rU   rV   rW   rX   rY   kwargs	__class__rC   rF   r\      sJ   
	

zCohere2Config.__init__c                 C   s   t dt | jS )NzTThe `sliding_window_pattern` attribute is deprecated and will be removed in v4.55.0.)warningswarnFutureWarningr@   rC   rE   rE   rF   r:      s
   z$Cohere2Config.sliding_window_patternc                 C   s
   || _ d S N)r@   )rD   valuerE   rE   rF   r:     s   
)r(   r)   r*   r+   r,   r-   Nr.   r)   r/   r0   Tr   r1   r2   Tr3   NFr4   r5   N)__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planr\   propertyr:   setter__classcell__rE   rE   ra   rF   r   1   sV    p


K
r   c                   @      e Zd ZdS )Cohere2RotaryEmbeddingNrh   ri   rj   rE   rE   rE   rF   rt         rt   c                   @   rs   )Cohere2LayerNormNru   rE   rE   rE   rF   rw     rv   rw   c                   @   s   e Zd ZdZddedee fddZ		ddej	de
ej	ej	f d	eej	 d
ee deej dee de
ej	eej	 ee
ej	  f fddZdS )Cohere2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperNconfig	layer_idxc                 C   s   t j  || _|| _t|d|j|j | _|j|j	 | _
| jd | _|j| _d| _|j| dkr4|jnd | _t j|j|j| j |jd| _t j|j|j	| j |jd| _t j|j|j	| j |jd| _t j|j| j |j|jd| _d S )NrZ   g      Tr=   )bias)nnModuler\   ry   rz   r^   rJ   rN   rZ   rO   num_key_value_groupsscalingrW   	is_causalrY   rX   LinearrV   q_projk_projv_projo_projrD   ry   rz   rE   rE   rF   r\     s*   
zCohere2Attention.__init__r#   position_embeddingsr$   past_key_valuecache_positionr`   returnc                 K   s2  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}| jd urGt|	|
||\}	}
|d ur\|||d}|	|
|| j
|\}
}t}| jjdkrjt| jj }|| |	|
||f| jsvdn| j| j| jd|\}}|jg |dR   }| |}||fS )Nr<   r   )sincosr   eagerr4   )dropoutr   rX   )shaperZ   r   view	transposer   r   rX   r   updaterz   r   ry   _attn_implementationr   trainingrW   r   reshape
contiguousr   )rD   r#   r   r$   r   r   r`   input_shapehidden_shapequery_states
key_statesvalue_statesr   r   cache_kwargsattention_interfaceattn_outputattn_weightsrE   rE   rF   forward.  s<   	
	

zCohere2Attention.forwardrf   )NN)rh   ri   rj   rk   r   r   intr\   torchTensortupler   
LongTensorr   r   r   rE   rE   rE   rF   rx     s(    rx   c                       s   e Zd Zdedef fddZeddd							dd
ejde	ejejf de
ej de
e de
e de
e de
ej dee de	eje
e	ejejf  f fddZ  ZS )Cohere2DecoderLayerry   rz   c                    s   t  || |j| | _d S rf   )r[   r\   rY   attention_typer   ra   rE   rF   r\   \  s   zCohere2DecoderLayer.__init__last_cache_positionz4.53.0)versionNFr#   r   r$   r   output_attentionsrS   r   r`   r   c              
   K   s`   |}	|  |}| jd|||||||d|\}
}| |}|	|
 | }|f}|r.||f7 }|S )ax  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
        )r#   r   r$   r   r   rS   r   NrE   )input_layernorm	self_attnmlp)rD   r#   r   r$   r   r   rS   r   r`   residualhidden_states_attentionself_attn_weightshidden_states_mlpoutputsrE   rE   rF   r   `  s&   



zCohere2DecoderLayer.forward)NNFFN)rh   ri   rj   r   r   r\   r   r   r   r   r   r   r?   r   r   r   FloatTensorr   rr   rE   rE   ra   rF   r   [  s6    
	
r   c                   @   s   e Zd ZeZdS )Cohere2PreTrainedModelN)rh   ri   rj   r   config_classrE   rE   rE   rF   r     s    r   c                       s   e Zd Zdef fddZ									ddeej deej deej dee	 d	eej
 d
ee dee dee deej dee defddZ  ZS )Cohere2Modelry   c                    s.   t  | t|j|jd| _t|d| _d S )N)rJ   eps)ry   )r[   r\   rw   rJ   rR   r'   rt   
rotary_emb)rD   ry   ra   rE   rF   r\     s   zCohere2Model.__init__Nr!   r$   position_idsr   r"   rS   r   output_hidden_statesr   flash_attn_kwargsr   c
              
   K   s  |d ur|n| j j}|d ur|n| j j}|d ur|n| j j}|d u |d uA r*td| jr9| jr9|r9td d}|d u rB| 	|}|rN|d u rN| jsNt
 }|	d u rj|d urZ| nd}tj|||jd  |jd}	|d u rs|	d}t| }ts| j |||	||d}td	i |td	i |d}|}| ||}|rd	nd }|rd	nd }| jD ](}|r||f7 }||f|||j ||||	d
|
}|d }|r||d f7 }q| |}|r||f7 }t||||dS )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fr   r<   )device)ry   input_embedsr$   r   r   r   )r>   r=   rE   )r   r$   r   r   rS   r   )last_hidden_stater   r#   
attentions)ry   r   r   rS   
ValueErrorgradient_checkpointingr   loggerwarning_oncer%   r   get_seq_lengthr   aranger   r   	unsqueeze
isinstancedictr	   r
   r   r&   r   r'   r   )rD   r!   r$   r   r   r"   rS   r   r   r   r   past_seen_tokenscausal_mask_mappingmask_kwargsr#   r   all_hidden_statesall_self_attnsdecoder_layerlayer_outputsrE   rE   rF   r     s   






zCohere2Model.forward)	NNNNNNNNN)rh   ri   rj   r   r\   r   r   r   r   r   r   r?   r   r   r   r   rr   rE   rE   ra   rF   r     sD    	
r   c                   @   rs   )Cohere2ForCausalLMNru   rE   rE   rE   rF   r     rv   r   )r   r   r   r   )7rc   typingr   r   r   torch.nnr|   torch.utils.checkpointcache_utilsr   r   configuration_utilsr   r   masking_utilsr	   r
   modeling_flash_attention_utilsr   modeling_outputsr   modeling_rope_utilsr   modeling_utilsr   processing_utilsr   utilsr   utils.deprecationr   cohere.modeling_coherer   r   r   r   r   r   r   r   gemma2.modeling_gemma2r   
get_loggerrh   r   r   rt   rw   r}   rx   r   r   r   r   __all__rE   rE   rE   rF   <module>   s8   (

 [HBl