o
    eia                     @   s  d dl mZ d dlZd dlmZ ddlmZ ddlmZm	Z	 ddl
mZmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZmZ ddlmZ ddlmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ e!0e1Z2G dd deZ3G dd de,Z4G dd de)Z5G dd de-Z6			d5dej7dej8dej8dej8dej8dB d e9d!e9dB d"e9dB d#e:ej8ej8f fd$d%Z;G d&d' d'e%Z<G d(d) d)eZ=G d*d+ d+e+Z>G d,d- d-e*Z?G d.d/ d/e&Z@G d0d1 d1e'ZAG d2d3 d3e(ZBg d4ZCdS )6    )CallableN   )ACT2FN)CacheDynamicCache)PreTrainedConfiglayer_type_validation)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSRopeParametersdynamic_rope_update)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargslogging)maybe_autocast   )GemmaAttentionGemmaForCausalLMGemmaForSequenceClassificationGemmaForTokenClassificationGemmaMLP
GemmaModelGemmaPreTrainedModelGemmaRMSNormGemmaRotaryEmbeddingapply_rotary_pos_emb	repeat_kvc                4       sr  e Zd ZdZdZdgZddddddddZdgdgfd	d
gd	gfd	gd	gfdZ																								 	d<d!edB d"edB d#edB d$edB d%edB d&edB d'edB d(e	dB d)edB d*e
dB d+edB d,edB d-edB d.edB d/edB d0edB d1eee	ef B dB d2edB d3e
dB d4edB d5edB d6ee	 dB d7e
dB d8e
dB d9edB f2 fd:d;Z  ZS )=Gemma2ConfigaK  
    This is the configuration class to store the configuration of a [`Gemma2Model`]. It is used to instantiate an Gemma2
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the Gemma2-7B.
    e.g. [google/gemma2-7b](https://huggingface.co/google/gemma2-7b)
    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 256000):
            Vocabulary size of the Gemma2 model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`Gemma2Model`]
        hidden_size (`int`, *optional*, defaults to 2304):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 9216):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 26):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 8):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*, defaults to 4):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details, check out [this
            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
            `num_attention_heads`.
        head_dim (`int`, *optional*, defaults to 256):
            The attention head dimension.
        hidden_activation (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
            The non-linear activation function (function or string) in the decoder. Will default to `"gelu_pytorch_tanh"`
            if not specified. `"gelu_pytorch_tanh"` uses an approximation of the `"gelu"` activation function.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length that this model might ever be used with.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 0):
            Padding token id.
        eos_token_id (`int`, *optional*, defaults to 1):
            End of stream token id.
        bos_token_id (`int`, *optional*, defaults to 2):
            Beginning of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
            Whether to tie weight embeddings
        rope_parameters (`RopeParameters`, *optional*):
            Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
            a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE
            with longer `max_position_embeddings`.
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        query_pre_attn_scalar (`float`, *optional*, defaults to 256):
            scaling factor used on the attention scores
        sliding_window (`int`, *optional*, defaults to 4096):
            in Gemma2, every other layer uses sliding window attention. This is the size of the sliding window.
        layer_types (`list`, *optional*):
            Attention pattern for each layer.
        final_logit_softcapping (`float`, *optional*, defaults to 30.0):
            scaling factor when applying tanh softcapping on the logits.
        attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
            scaling factor when applying tanh softcapping on the attention scores.
        use_bidirectional_attention (`bool`, *optional*):
            If True, the model will attend to all text tokens instead of using a causal mask.

    ```python
    >>> from transformers import Gemma2Model, Gemma2Config
    >>> # Initializing a Gemma2 gemma2-7b style configuration
    >>> configuration = Gemma2Config()
    >>> # Initializing a model from the gemma2-7b style configuration
    >>> model = Gemma2Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```gemma2past_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnorm   	   $              gelu_pytorch_tanh    {Gz?ư>Tr      r   NF                 >@      I@
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_headshead_dimhidden_activationmax_position_embeddingsinitializer_rangerms_norm_eps	use_cachepad_token_ideos_token_idbos_token_idtie_word_embeddingsrope_parametersattention_biasattention_dropoutquery_pre_attn_scalarsliding_windowlayer_typesfinal_logit_softcappingattn_logit_softcappinguse_bidirectional_attentionc                    s   || _ || _|| _|| _|| _|	| _|| _|| _|| _|| _	|| _
|| _|
| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _| jd u rXdd t| jD | _t| j| j || _t jdi | d S )Nc                 S   s$   g | ]}t |d  d rdndqS )r:   r   sliding_attentionfull_attention)bool).0i r]   g/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/gemma2/modular_gemma2.py
<listcomp>   s    z)Gemma2Config.__init__.<locals>.<listcomp>r]   )rK   rM   rL   rN   r?   rG   r@   rA   rB   rC   rE   rD   rH   rI   rJ   rP   rQ   rF   rR   rS   rU   rV   rT   rW   ranger   rO   super__init__)selfr?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   kwargs	__class__r]   r^   rb      s>   
zGemma2Config.__init__)r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   Tr   r:   r   TNFr;   r5   r<   Nr=   r>   N)__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planintstrfloatrZ   r   dictlistrb   __classcell__r]   r]   re   r^   r#   6   s    Q


	

r#   c                   @      e Zd ZdS )Gemma2RMSNormNrg   rh   ri   r]   r]   r]   r^   rv          rv   c                       s   e Zd Z fddZ  ZS )	Gemma2MLPc                    s   t  | t|j | _d S N)ra   rb   r   rF   act_fnrc   configre   r]   r^   rb      s   zGemma2MLP.__init__)rg   rh   ri   rb   rt   r]   r]   re   r^   ry      s    ry   c                   @   s0   e Zd ZddefddZe edd ZdS )Gemma2RotaryEmbeddingNr}   c                 C   s   t j  |j| _|j| _|| _| jjd | _| j	}| jdkr$t
| j }|| j|\}| _| jd|dd | jd| dd d S )N	rope_typedefaultinv_freqF)
persistentoriginal_inv_freq)nnModulerb   rG   max_seq_len_cachedoriginal_max_seq_lenr}   rO   r   compute_default_rope_parametersr   attention_scalingregister_bufferclone)rc   r}   devicerope_init_fnr   r]   r]   r^   rb      s   


zGemma2RotaryEmbedding.__init__c           
      C   s   | j d d d d f  |jd dd|j}|d d d d d f  }t|jjtr6|jjdkr6|jjnd}t	|dd+ | |  
dd}tj||fdd	}| | j }| | j }	W d    n1 slw   Y  |j|jd
|	j|jd
fS )Nr   r:   mpscpuF)device_typeenabledr   )dimdtype)r   rq   expandshapetor   
isinstancetyperp   r   	transposetorchcatcosr   sinr   )
rc   xposition_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   r]   r]   r^   forward   s   0&zGemma2RotaryEmbedding.forwardrz   )	rg   rh   ri   r#   rb   r   no_gradr   r   r]   r]   r]   r^   r~      s
    r~   r;   modulequerykeyvaluer+   dropoutscalingsoftcapreturnc                 K   s   |d u r	| j d }t|| j}	t|| j}
t||	dd| }|d ur2|| }t|}|| }|d ur:|| }tjj	|dtj
d|j}tjj||| jd}t||
}|dd }||fS )N      r   r   r   )r   r   )ptrainingr:   )rE   r"   num_key_value_groupsr   matmulr   tanhr   
functionalsoftmaxfloat32r   r   r   r   
contiguous)r   r   r   r   r+   r   r   r   rd   
key_statesvalue_statesattn_weightsattn_outputr]   r]   r^   eager_attention_forward  s    

r   c                       s   e Zd Zdedef fddZ				ddejdeejejf dB dejdB d	e	dB d
ej
dB dee deejejdB eej dB f fddZ  ZS )Gemma2Attentionr}   	layer_idxc                    sx   t |dr
|j| nd | _t || | jj| _| jj| _t|dd | _	|j
d | _| jdkr7|j| _d S d | _d S )NrT   rW   Fr   rX   )hasattrrT   
layer_typera   rb   r}   rV   rQ   getattr	is_causalrR   r   rS   rc   r}   r   re   r]   r^   rb   (  s   

 zGemma2Attention.__init__Nr*   position_embeddingsr+   r%   cache_positionrd   r   c                 K   s   |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}t|	|
||\}	}
|d urW|||d}||
|| j	|\}
}t
| jjt}|| |	|
||f| jrl| jnd| j| j| jd|\}}|jg |dR   }| |}||fS )Nr   r:   r   )r   r   r   r;   )r   r   rS   r   )r   rE   q_projviewr   k_projv_projr!   updater   r   get_interfacer}   _attn_implementationr   r   rQ   r   rS   rV   reshaper   o_proj)rc   r*   r   r+   r%   r   rd   input_shapehidden_shapequery_statesr   r   r   r   cache_kwargsattention_interfacer   r   r]   r]   r^   r   2  s<   	


zGemma2Attention.forward)NNNN)rg   rh   ri   r#   ro   rb   r   Tensortupler   
LongTensorr   r   r   rt   r]   r]   re   r^   r   '  s*    r   c                       s   e Zd Zdedef fddZ					ddejdeejejf dB dejdB d	ej	dB d
e
dB dej	dB deejeejejf dB f fddZ  ZS )Gemma2DecoderLayerr}   r   c                    s   t    |j| _|| _|j| | _t||d| _t|| _	t
|j|jd| _t
|j|jd| _t
|j|jd| _t
|j|jd| _d S )N)r}   r   )eps)ra   rb   r@   r}   rT   attention_typer   	self_attnry   mlprv   rI   input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormr   re   r]   r^   rb   a  s   

zGemma2DecoderLayer.__init__Nr*   r   r+   r   r%   r   r   c           
   	   K   sp   |}|  |}| jd||||||d|\}}	| |}|| }|}| |}| |}| |}|| }|S )N)r*   r   r+   r   r%   r   r]   )r   r   r   r   r   r   )
rc   r*   r   r+   r   r%   r   rd   residual_r]   r]   r^   r   n  s(   



	


zGemma2DecoderLayer.forward)NNNNN)rg   rh   ri   r#   ro   rb   r   r   r   r   r   FloatTensorr   rt   r]   r]   re   r^   r   `  s,    	r   c                   @   ru   )Gemma2PreTrainedModelNrw   r]   r]   r]   r^   r     rx   r   c                       s   e Zd Zdef fddZ							ddejdB dejdB dejdB dedB d	ej	dB d
e
dB dejdB dee defddZ  ZS )Gemma2Modelr}   c                    s:   t    t fddt jD | _t | _d S )Nc                    s   g | ]}t  |qS r]   )r   )r[   r   r}   r]   r^   r_     s    z(Gemma2Model.__init__.<locals>.<listcomp>)	ra   rb   r   
ModuleListr`   rB   r-   r~   
rotary_embr|   re   r   r^   rb     s
   zGemma2Model.__init__Nr(   r+   r   r%   r)   rJ   r   rd   r   c              	   K   sL  |d u |d uA rt d|d u r| |}|r!|d u r!t| jd}|d u r=|d ur-| nd}	tj|	|	|jd  |jd}|d u rF|	d}t
| }
tsf| j|||||d}tdi |tdi |d}
|}| ||}tj| jjd |jd	}|| }| jd | jj D ]}||f|
|j ||||d
|}q| |}t||dS )Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r:   )r   )r}   r)   r+   r   r%   r   )rY   rX   g      ?r   )r+   r   r   r%   r   )last_hidden_stater%   r]   )
ValueErrorr,   r   r}   get_seq_lengthr   aranger   r   	unsqueezer   rr   r	   r
   r   tensorr@   r   r-   rB   r   r.   r   )rc   r(   r+   r   r%   r)   rJ   r   rd   past_seen_tokenscausal_mask_mappingmask_kwargsr*   r   
normalizerdecoder_layerr]   r]   r^   r     sZ   




zGemma2Model.forward)NNNNNNN)rg   rh   ri   r#   rb   r   r   r   r   r   rZ   r   r   r   r   rt   r]   r]   re   r^   r     s8    		
r   c                       s   e Zd Z fddZ									ddejdB dejdB dejdB dedB d	ejdB d
ejdB de	dB dejdB de
ejB dee defddZ  ZS )Gemma2ForCausalLMc                    s"   t  | t|| _|   d S rz   )ra   rb   r   model	post_initr|   re   r]   r^   rb     s   
zGemma2ForCausalLM.__init__Nr   r(   r+   r   r%   r)   labelsrJ   r   logits_to_keeprd   r   c
              
   K   s   | j d|||||||d|
}|j}t|	trt|	 dn|	}| |dd|ddf }| jjdurE|| jj }t	|}|| jj }d}|durW| j
||| jfi |
}t|||j|j|jdS )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma2ForCausalLM

        >>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)r(   r+   r   r%   r)   rJ   r   N)losslogitsr%   r*   
attentionsr]   )r   r   r   ro   slicelm_headr}   rU   r   r   loss_functionr?   r   r%   r*   r   )rc   r(   r+   r   r%   r)   r   rJ   r   r   rd   outputsr*   slice_indicesr   r   r]   r]   r^   r     s8   
zGemma2ForCausalLM.forward)	NNNNNNNNr   )rg   rh   ri   rb   r   r   r   r   r   rZ   ro   r   r   r   r   rt   r]   r]   re   r^   r     sD    	
r   c                   @   ru   )Gemma2ForSequenceClassificationNrw   r]   r]   r]   r^   r  /  rx   r  c                   @   ru   )Gemma2ForTokenClassificationNrw   r]   r]   r]   r^   r  3  rx   r  )r#   r   r   r   r  r  )r;   NN)Dcollections.abcr   r   torch.nnr   activationsr   cache_utilsr   r   configuration_utilsr   r   masking_utilsr	   r
   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   r   r   modeling_utilsr   processing_utilsr   utilsr   r   utils.genericr   gemma.modeling_gemmar   r   r   r   r   r   r   r   r    r!   r"   
get_loggerrg   loggerr#   rv   ry   r~   r   r   rq   r   r   r   r   r   r   r   r  r  __all__r]   r]   r]   r^   <module>   sh   4
 %'

"92SF