o
    iJa                     @   s  d dl mZmZmZ d dlZd dlmZ ddlmZ ddl	m
Z
mZ ddlmZmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- e.e/Z0G dd deZ1G dd de*Z2G dd de'Z3G dd de+Z4			d4dej5dej6dej6dej6deej6 de7d ee7 d!ee7 d"e8ej6ej6f fd#d$Z9G d%d& d&e#Z:G d'd( d(eZ;G d)d* d*e)Z<G d+d, d,e(Z=G d-d. d.e$Z>G d/d0 d0e%Z?G d1d2 d2e&Z@g d3ZAdS )5    )CallableOptionalUnionN   )ACT2FN)CacheDynamicCache)PretrainedConfiglayer_type_validation)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargslogging)deprecate_kwarg   )GemmaAttentionGemmaForCausalLMGemmaForSequenceClassificationGemmaForTokenClassificationGemmaMLP
GemmaModelGemmaPreTrainedModelGemmaRMSNormGemmaRotaryEmbeddingapply_rotary_pos_emb	repeat_kvc                       s   e Zd ZdZdZdgZddddddddZdgdgfd	d
gd	gfd	gd	gfdZ																							 	!d$ fd"d#	Z  Z	S )%Gemma2Configa  
    This is the configuration class to store the configuration of a [`Gemma2Model`]. It is used to instantiate an Gemma2
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the Gemma2-7B.
    e.g. [google/gemma2-7b](https://huggingface.co/google/gemma2-7b)
    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.
    Args:
        vocab_size (`int`, *optional*, defaults to 256000):
            Vocabulary size of the Gemma2 model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`Gemma2Model`]
        hidden_size (`int`, *optional*, defaults to 2304):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 9216):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 26):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 8):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*, defaults to 4):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details, check out [this
            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
            `num_attention_heads`.
        head_dim (`int`, *optional*, defaults to 256):
            The attention head dimension.
        hidden_activation (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
            The non-linear activation function (function or string) in the decoder. Will default to `"gelu_pytorch_tanh"`
            if not specified. `"gelu_pytorch_tanh"` uses an approximation of the `"gelu"` activation function.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length that this model might ever be used with.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 0):
            Padding token id.
        eos_token_id (`int`, *optional*, defaults to 1):
            End of stream token id.
        bos_token_id (`int`, *optional*, defaults to 2):
            Beginning of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
            Whether to tie weight embeddings
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        query_pre_attn_scalar (`float`, *optional*, defaults to 256):
            scaling factor used on the attention scores
        sliding_window (`int`, *optional*, defaults to 4096):
            in Gemma2, every other layer uses sliding window attention. This is the size of the sliding window.
        layer_types (`list`, *optional*):
            Attention pattern for each layer.
        final_logit_softcapping (`float`, *optional*, defaults to 30.0):
            scaling factor when applying tanh softcapping on the logits.
        attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
            scaling factor when applying tanh softcapping on the attention scores.

    ```python
    >>> from transformers import Gemma2Model, Gemma2Config
    >>> # Initializing a Gemma2 gemma2-7b style configuration
    >>> configuration = Gemma2Config()
    >>> # Initializing a model from the gemma2-7b style configuration
    >>> model = Gemma2Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```gemma2past_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnorm   	   $              gelu_pytorch_tanh    {Gz?ư>Tr      r        @F           N      >@      I@c                    s   t  jd||||d| || _|	| _|| _|| _|| _|| _|| _|| _	|
| _
|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _| jd u rZdd t| jD | _t| j| j d S )N)pad_token_idbos_token_ideos_token_idtie_word_embeddingsc                 S   s$   g | ]}t |d  d rdndqS )r9   r   sliding_attentionfull_attention)bool).0i rH   f/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/transformers/models/gemma2/modular_gemma2.py
<listcomp>   s    z)Gemma2Config.__init__.<locals>.<listcomp>rH   )super__init__
vocab_sizemax_position_embeddingshidden_sizeintermediate_sizenum_hidden_layersnum_attention_headshead_dimnum_key_value_headsinitializer_rangerms_norm_eps	use_cache
rope_thetaattention_biasattention_dropouthidden_activationquery_pre_attn_scalarsliding_windowfinal_logit_softcappingattn_logit_softcappinglayer_typesranger
   )selfrM   rO   rP   rQ   rR   rT   rS   r[   rN   rU   rV   rW   r?   rA   r@   rB   rX   rY   rZ   r\   r]   r`   r^   r_   kwargs	__class__rH   rI   rL      sB   
zGemma2Config.__init__)r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   Tr   r9   r   Tr:   Fr;   r4   r<   Nr=   r>   )
__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planrL   __classcell__rH   rH   rd   rI   r"   2   sR    L


r"   c                   @      e Zd ZdS )Gemma2RMSNormNrf   rg   rh   rH   rH   rH   rI   rp          rp   c                       s   e Zd Z fddZ  ZS )	Gemma2MLPc                    s   t  | t|j | _d S N)rK   rL   r   r[   act_fnrb   configrd   rH   rI   rL      s   zGemma2MLP.__init__)rf   rg   rh   rL   rn   rH   rH   rd   rI   rs      s    rs   c                   @   ro   )Gemma2RotaryEmbeddingNrq   rH   rH   rH   rI   rx      rr   rx   r;   modulequerykeyvaluer*   dropoutscalingsoftcapreturnc                 K   s   |d u r	| j d }t|| j}	t|| j}
t||	dd| }|d ur2|| }t|}|| }|d urM|d d d d d d d |	jd f }|| }tj	j
|dtjd|j}tj	j||| jd}t||
}|dd }||fS )	N      r   r   )dimdtype)ptrainingr9   )rS   r!   num_key_value_groupstorchmatmul	transposetanhshapenn
functionalsoftmaxfloat32tor   r}   r   
contiguous)ry   rz   r{   r|   r*   r}   r~   r   rc   
key_statesvalue_statesattn_weightscausal_maskattn_outputrH   rH   rI   eager_attention_forward   s"   

&r   c                       s   e Zd Zdedef fddZedddd				dd
ejde	ejejf de
ej de
e de
ej dee de	eje
ej e
e	ej  f fddZ  ZS )Gemma2Attentionrw   	layer_idxc                    sX   t  || | jj| _| jj| _d| _|jd | _|j| dkr'|j	| _	d S d | _	d S )NTr   rC   )
rK   rL   rw   r_   rZ   	is_causalr\   r~   r`   r]   rb   rw   r   rd   rH   rI   rL     s   

$zGemma2Attention.__init__past_key_valuer$   4.58new_nameversionNr)   position_embeddingsr*   cache_positionrc   r   c                 K   s,  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}t|	|
||\}	}
|d urW|||d}||
|| j	|\}
}t
}| jjdkret| jj }|| |	|
||f| jrr| jnd| j| j| jd|\}}|jg |dR   }| |}||fS )Nr   r9   r   )sincosr   eagerr;   )r}   r~   r]   r   )r   rS   q_projviewr   k_projv_projr    updater   r   rw   _attn_implementationr   r   rZ   r~   r]   r_   reshaper   o_proj)rb   r)   r   r*   r$   r   rc   input_shapehidden_shapequery_statesr   r   r   r   cache_kwargsattention_interfacer   r   rH   rH   rI   forward	  s<   



zGemma2Attention.forward)NN)rf   rg   rh   r"   intrL   r   r   Tensortupler   r   
LongTensorr   r   r   rn   rH   rH   rd   rI   r      s(    r   c                       s   e Zd Zdedef fddZedddd							
	
		ddejde	ejejf de
ej de
ej de
e de
e de
e de
ej de	eje
e	ejejf  f fddZ  ZS )Gemma2DecoderLayerrw   r   c                    s   t    |j| _|| _|j| | _t||d| _t|| _	t
|j|jd| _t
|j|jd| _t
|j|jd| _t
|j|jd| _d S )N)rw   r   )eps)rK   rL   rO   rw   r`   attention_typer   	self_attnrs   mlprp   rV   input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormr   rd   rH   rI   rL   9  s   

zGemma2DecoderLayer.__init__r   r$   r   r   NFr)   r   r*   position_idsoutput_attentionsrW   r   r   c	                 K   s   |}
|  |}| jd||||||||d|	\}}| |}|
| }|}
| |}| |}| |}|
| }|f}|rB||f7 }|S )N)r)   r   r*   r   r$   r   rW   r   rH   )r   r   r   r   r   r   )rb   r)   r   r*   r   r$   r   rW   r   rc   residualself_attn_weightsoutputsrH   rH   rI   r   F  s2   
	





zGemma2DecoderLayer.forward)NNNFFN)rf   rg   rh   r"   r   rL   r   r   r   r   r   r   r   rE   FloatTensorr   rn   rH   rH   rd   rI   r   8  s8    	r   c                   @   ro   )Gemma2PreTrainedModelNrq   rH   rH   rH   rI   r   t  rr   r   c                       s   e Zd Zdef fddZ									ddeej deej deej dee	 d	eej
 d
ee dee dee deej dee defddZ  ZS )Gemma2Modelrw   c                    s0   t    t fddt jD | _d S )Nc                    s   g | ]}t  |qS rH   )r   )rF   r   rw   rH   rI   rJ   |  s    z(Gemma2Model.__init__.<locals>.<listcomp>)rK   rL   r   
ModuleListra   rQ   r,   rv   rd   r   rI   rL   y  s   
zGemma2Model.__init__Nr'   r*   r   r$   r(   rW   r   output_hidden_statesr   rc   r   c
                 K   s  |d ur|n| j j}|d ur|n| j j}|d ur|n| j j}|d u |d uA r*td| jr9| jr9|r9td d}|d u rB| 	|}|rQ|d u rQ| jsQt
| j d}|	d u rm|d ur]| nd}tj|||jd  |jd}	|d u rv|	d}t| }ts| j |||	||d}tdi |tdi |d	}|}| ||}tj| j jd
 |jd}|| }|rdnd }|rdnd }| jd | j j D ])}|r||f7 }||f|||j |||||	d|
}|d }|r||d f7 }q| |}|r||f7 }t||||dS )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fr   r   r9   )device)rw   input_embedsr*   r   r$   r   )rD   rC   g      ?)r   rH   )r   r*   r   r$   r   rW   r   )last_hidden_stater$   r)   
attentions)rw   r   r   rW   
ValueErrorgradient_checkpointingr   loggerwarning_oncer+   r   get_seq_lengthr   aranger   r   	unsqueeze
isinstancedictr   r   
rotary_embtensorrO   r   r,   rQ   r   r-   r   )rb   r'   r*   r   r$   r(   rW   r   r   r   rc   past_seen_tokenscausal_mask_mappingmask_kwargsr)   r   
normalizerall_hidden_statesall_self_attnsdecoder_layerlayer_outputsrH   rH   rI   r     s   



	

zGemma2Model.forward)	NNNNNNNNN)rf   rg   rh   r"   rL   r   r   r   r   r   r   rE   r   r   r   r   rn   rH   rH   rd   rI   r   x  sD    	
r   c                       s   e Zd Z fddZ											ddeej deej deej dee d	eej	 d
eej dee
 dee
 dee
 deej deeejf defddZ  ZS )Gemma2ForCausalLMc                    s"   t  | t|| _|   d S rt   )rK   rL   r   model	post_initrv   rd   rH   rI   rL     s   
zGemma2ForCausalLM.__init__Nr   r'   r*   r   r$   r(   labelsrW   r   r   r   logits_to_keepr   c                 K   s   |dur|n| j j}|	dur|	n| j j}	| jd||||||||	|
d	|}|j}t|tr4t| dn|}| |dd|ddf }| j j	dur[|| j j	 }t
|}|| j j	 }d}|durm| j||| jfi |}t|||j|j|jdS )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma2ForCausalLM

        >>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```N)	r'   r*   r   r$   r(   rW   r   r   r   )losslogitsr$   r)   r   rH   )rw   r   r   r   r   r   r   slicelm_headr^   r   r   loss_functionrM   r   r$   r)   r   )rb   r'   r*   r   r$   r(   r   rW   r   r   r   r   rc   r   r)   slice_indicesr   r   rH   rH   rI   r     sB   !

zGemma2ForCausalLM.forward)NNNNNNNNNNr   )rf   rg   rh   rL   r   r   r   r   r   r   rE   r   r   r   r   rn   rH   rH   rd   rI   r     sL    	
r   c                   @   ro   )Gemma2ForSequenceClassificationNrq   rH   rH   rH   rI   r   <  rr   r   c                   @   ro   )Gemma2ForTokenClassificationNrq   rH   rH   rH   rI   r   @  rr   r   )r"   r   r   r   r   r   )r;   NN)Btypingr   r   r   r   torch.nnr   activationsr   cache_utilsr   r   configuration_utilsr	   r
   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   modeling_utilsr   processing_utilsr   utilsr   r   utils.deprecationr   gemma.modeling_gemmar   r   r   r   r   r   r   r   r   r    r!   
get_loggerrf   r   r"   rp   rs   rx   Moduler   floatr   r   r   r   r   r   r   r   r   __all__rH   rH   rH   rI   <module>   sf   4
 


#8<uO