o
    eiH                     @   sj  d dl mZ d dlZd dlmZ ddlmZmZ ddlm	Z	m
Z
 ddlmZmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& e'e(Z)G dd de	Z*G dd de"Z+G dd de Z,G dd deZ-G dd deZ.G dd de!Z/G dd de&Z0G dd deZ1g d Z2dS )!    )CallableN   )CacheDynamicCache)PreTrainedConfiglayer_type_validation)create_causal_mask!create_sliding_window_causal_mask)BaseModelOutputWithPast)RopeParametersdynamic_rope_update)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargslogging)maybe_autocast   )CohereAttentionCohereDecoderLayerCohereForCausalLMCohereLayerNormCoherePreTrainedModelCohereRotaryEmbeddingapply_rotary_pos_embeager_attention_forward)Gemma2Modelc                ,       sJ  e Zd ZdZdZdgZddddddddZdgdgfd	d
gd	gfd	gd	gfdZ																					d4dedB dedB dedB d e	dB d!edB d"edB d#edB d$e
dB d%edB d&e	dB d'edB d(edB d)edB d*edB d+edB d,edB d-eee
ef B dB d.edB d/e	dB d0edB d1ee
 dB f* fd2d3Z  ZS )5Cohere2Configac  
    This is the configuration class to store the configuration of a [`CohereModel`]. It is used to instantiate an Cohere
    model according to the specified arguments, defining the model architecture.

    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information. Instantiating a configuration
    with the defaults will yield a similar configuration to that of the [CohereForAI/c4ai-command-r-v01](https://huggingface.co/CohereForAI/c4ai-command-r-v01) model.


    Args:
        vocab_size (`int`, *optional*, defaults to 256000):
            Vocabulary size of the Cohere model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`CohereModel`]
        hidden_size (`int`, *optional*, defaults to 8192):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 22528):
            Dimension of the MLP representations.
        logit_scale (`float`, *optional*, defaults to 0.0625):
            The scaling factor for the output logits.
        num_hidden_layers (`int`, *optional*, defaults to 40):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 64):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details, check out [this
            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
            `num_attention_heads`.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length that this model might ever be used with.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the layer normalization.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 0):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 5):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 255001):
            End of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
            Whether to tie weight embeddings
        rope_parameters (`RopeParameters`, *optional*):
            Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
            a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE
            with longer `max_position_embeddings`.
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        sliding_window (`int`, *optional*, defaults to 4096):
            Size of the sliding window attention context.
        layer_types (`list`, *optional*):
            Attention pattern for each layer.

    ```python
    >>> from transformers import Cohere2Model, Cohere2Config

    >>> # Initializing a Cohere Nextmodel configuration
    >>> configuration = Cohere2Config()

    >>> # Initializing a model from the Cohere2 configuration
    >>> model = Cohere2Model(configuration) # doctest: +SKIP

    >>> # Accessing the model configuration
    >>> configuration = model.config # doctest: +SKIP
    ```
    cohere2past_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnorm       X        ?(   @   Nsilu{Gz?h㈵>Tr       F           
vocab_sizehidden_sizeintermediate_sizelogit_scalenum_hidden_layersnum_attention_headsnum_key_value_heads
hidden_actmax_position_embeddingsinitializer_rangelayer_norm_eps	use_cachepad_token_idbos_token_ideos_token_idtie_word_embeddingsrope_parametersattention_biasattention_dropoutsliding_windowlayer_typesc                    s   | _ |	 _| _| _| _| _| _|d u r|}| _| _|
 _	| _
| _| _| _| _| _||  _| _| _| _| _|dd _ jd u rgt dd _ fddt jD  _t j j | _t jdi | d S )Nsliding_window_pattern   c                    s&   g | ]}t |d   j rdndqS )   sliding_attentionfull_attention)bool_sliding_window_pattern).0iself i/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/cohere2/modular_cohere2.py
<listcomp>   s    z*Cohere2Config.__init__.<locals>.<listcomp>rU   )r5   r=   r6   r8   r7   r9   r:   r;   r<   r>   r?   r@   rF   rG   rH   rI   head_dimrA   rB   rC   rD   getrP   getattrranger   rE   super__init__)rT   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   kwargs	__class__rS   rV   r]      s@   


zCohere2Config.__init__)r(   r)   r*   r+   r,   r-   Nr.   r)   r/   r0   Tr   r1   r2   TNFr3   r4   N)__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planintfloatstrrO   r   dictlistr]   __classcell__rU   rU   r_   rV   r   0   s    M


	

r   c                   @   s    e Zd Ze edd ZdS )Cohere2RotaryEmbeddingc           
      C   s   | j d d d d f  |jd dd}|d d d d d f  }t|jjtr2|jjdkr2|jjnd}t|dd* | |  	dd}t
j|ddd	}| | j }| | j }	W d    n1 sgw   Y  |j|jd
|	j|jd
fS )Nr   rL   mpscpuF)device_typeenabledr   )dim)dtype)inv_freqrj   expandshape
isinstancedevicetyperk   r   	transposetorchrepeat_interleavecosattention_scalingsintorv   )
rT   xposition_idsinv_freq_expandedposition_ids_expandedrs   freqsembr   r   rU   rU   rV   forward   s   (&zCohere2RotaryEmbedding.forwardN)ra   rb   rc   r~   no_gradr   r   rU   rU   rU   rV   ro      s    ro   c                   @      e Zd ZdS )Cohere2LayerNormNra   rb   rc   rU   rU   rU   rV   r          r   c                   @   s   e Zd ZdZddededB fddZ		ddejde	ejejf d	ejdB d
e
dB dejdB dee de	ejejdB e	ej dB f fddZdS )Cohere2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperNconfig	layer_idxc                 C   s   t j|  || _|| _t|d|j|j | _|j|j	 | _
| jd | _|j| _d| _t|dr5|j| nd }|dkr>|jnd | _t j|j|j| j |jd| _t j|j|j	| j |jd| _t j|j|j	| j |jd| _t j|j| j |j|jd| _d S )NrX   g      TrI   rM   )bias)nnModuler]   r   r   rZ   r6   r:   rX   r;   num_key_value_groupsscalingrG   	is_causalhasattrrI   rH   LinearrF   q_projk_projv_projo_proj)rT   r   r   
layer_typerU   rU   rV   r]      s,   zCohere2Attention.__init__r#   position_embeddingsr$   r   cache_positionr^   returnc                 K   s&  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}| jd urGt|	|
||\}	}
|d ur\|||d}|	|
|| j
|\}
}t| jjt}|| |	|
||f| jspdn| j| j| jd|\}}|jg |dR   }| |}||fS )Nrp   rL   r   )r   r   r   r3   )dropoutr   rH   )ry   rX   r   viewr}   r   r   rH   r   updater   r   get_interfacer   _attn_implementationr   trainingrG   r   reshape
contiguousr   )rT   r#   r   r$   r   r   r^   input_shapehidden_shapequery_states
key_statesvalue_statesr   r   cache_kwargsattention_interfaceattn_outputattn_weightsrU   rU   rV   r     s<   	
	

zCohere2Attention.forwardN)NN)ra   rb   rc   rd   r   ri   r]   r~   Tensortupler   
LongTensorr   r   r   rU   rU   rU   rV   r      s(    r   c                       s   e Zd Zdedef fddZ					ddejdeejejf dB d	ejdB d
e	dB de
dB dejdB dee deejeejejf dB f fddZ  ZS )Cohere2DecoderLayerr   r   c                    s   t  || |j| | _d S r   )r\   r]   rI   attention_type)rT   r   r   r_   rU   rV   r]   5  s   zCohere2DecoderLayer.__init__NFr#   r   r$   r   r@   r   r^   r   c              	   K   sJ   |}|  |}| jd||||||d|\}	}
| |}||	 | }|S )N)r#   r   r$   r   r@   r   rU   )input_layernorm	self_attnmlp)rT   r#   r   r$   r   r@   r   r^   residualhidden_states_attention_hidden_states_mlprU   rU   rV   r   9  s   




zCohere2DecoderLayer.forward)NNNFN)ra   rb   rc   r   ri   r]   r~   r   r   r   rO   r   r   r   FloatTensorr   rn   rU   rU   r_   rV   r   4  s0    	r   c                   @   s   e Zd ZU eed< dS )Cohere2PreTrainedModelr   N)ra   rb   rc   r   __annotations__rU   rU   rU   rV   r   T  s   
 r   c                       s   e Zd Zdef fddZ							ddejdB dejdB dejdB dedB d	ej	dB d
e
dB dejdB dee defddZ  ZS )Cohere2Modelr   c                    s"   t  | t|j|jd| _d S )N)r6   eps)r\   r]   r   r6   r?   r'   )rT   r   r_   rU   rV   r]   Y  s   zCohere2Model.__init__Nr!   r$   r   r   r"   r@   r   r^   r   c              
   K   s"  |d u |d uA rt d|d u r| |}|r!|d u r!t| jd}|d u r=|d ur-| nd}	tj|	|	|jd  |jd}|d u rF|	d}t
| }
tsf| j|||||d}td
i |td
i |d}
|}| ||}| jD ]}||f|
|j |||||d|}qq| |}t||d	S )Nz:You must specify exactly one of input_ids or inputs_embeds)r   r   rL   )r{   )r   r"   r$   r   r   r   )rN   rM   )r$   r   r   r@   r   r   )last_hidden_stater   rU   )
ValueErrorr%   r   r   get_seq_lengthr~   arangery   r{   	unsqueezerz   rl   r   r	   
rotary_embr&   r   r'   r
   )rT   r!   r$   r   r   r"   r@   r   r^   past_seen_tokenscausal_mask_mappingmask_kwargsr#   r   decoder_layerrU   rU   rV   r   ]  sX   

	

zCohere2Model.forward)NNNNNNN)ra   rb   rc   r   r]   r~   r   r   r   r   rO   r   r   r
   r   rn   rU   rU   r_   rV   r   X  s8    	
r   c                   @   r   )Cohere2ForCausalLMNr   rU   rU   rU   rV   r     r   r   )r   r   r   r   )3collections.abcr   r~   torch.nnr   cache_utilsr   r   configuration_utilsr   r   masking_utilsr   r	   modeling_outputsr
   modeling_rope_utilsr   r   modeling_utilsr   processing_utilsr   utilsr   r   utils.genericr   cohere.modeling_coherer   r   r   r   r   r   r   r   gemma2.modeling_gemma2r   
get_loggerra   loggerr   ro   r   r   r   r   r   r   __all__rU   rU   rU   rV   <module>   s2   (

 'I E