o
    iP                     @   sN  d dl mZmZmZ d dlZd dlmZ ddlmZ ddlm	Z	m
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZmZ ddlmZmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ e%,e-Z.edG dd dej/Z0G dd dej/Z1dd Z2d@ddZ3G dd dej/Z4dej5d e6d!ej5fd"d#Z7	$dAd%ej/d&ej5d'ej5d(ej5d)eej5 d*e8d+e8d,e e" fd-d.Z9G d/d0 d0ej/Z:G d1d2 d2eZ;e#G d3d4 d4eZ<e#G d5d6 d6e<Z=e#G d7d8 d8e<eZ>G d9d: d:ee<Z?G d;d< d<ee<Z@G d=d> d>ee<ZAg d?ZBdS )B    )CallableOptionalUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask)GenericForQuestionAnswering GenericForSequenceClassificationGenericForTokenClassificationGradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)deprecate_kwarg)check_model_inputs   )LlamaConfigRMSNormc                       s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	LlamaRMSNormư>c                    s&   t    tt|| _|| _dS )z;
        LlamaRMSNorm is equivalent to T5LayerNorm
        N)super__init__r   	Parametertorchonesweightvariance_epsilon)selfhidden_sizeeps	__class__ e/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.pyr$   6   s   

zLlamaRMSNorm.__init__c                 C   sJ   |j }|tj}|djddd}|t|| j  }| j|| S )N   T)keepdim)	dtypetor&   float32powmeanrsqrtr)   r(   )r*   hidden_statesinput_dtypevariancer/   r/   r0   forward>   s
   zLlamaRMSNorm.forwardc                 C   s   t | jj d| j S )Nz, eps=)tupler(   shaper)   )r*   r/   r/   r0   
extra_reprE   s   zLlamaRMSNorm.extra_repr)r"   )__name__
__module____qualname__r$   r=   r@   __classcell__r/   r/   r-   r0   r!   4   s    r!   c                       sD   e Zd ZU ejed< ddef fddZe e	dd Z
  ZS )	LlamaRotaryEmbeddinginv_freqNconfigc                    s   t    t|drt|jtr|jd|jd| _nd| _|j| _	|j| _
|| _t| j | _| | j|\}| _| jd|dd | j| _d S )Nrope_scaling	rope_typetypedefaultrF   F)
persistent)r#   r$   hasattr
isinstancerH   dictgetrI   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrG   r   rope_init_fnattention_scalingregister_bufferrF   original_inv_freq)r*   rG   devicerF   r-   r/   r0   r$   L   s   
zLlamaRotaryEmbedding.__init__c           
      C   s   | j d d d d f  |jd dd|j}|d d d d d f  }t|jjtr6|jjdkr6|jjnd}t	j
|dd+ | |  dd}t	j||fdd	}| | j }| | j }	W d    n1 smw   Y  |j|jd
|	j|jd
fS )Nr   r2   r   mpscpuF)device_typeenabledr1   dim)r4   )rF   floatexpandr?   r5   rX   rN   rJ   strr&   autocast	transposecatcosrU   sinr4   )
r*   xposition_idsinv_freq_expandedposition_ids_expandedr[   freqsembre   rf   r/   r/   r0   r=   ]   s   0&zLlamaRotaryEmbedding.forwardN)rA   rB   rC   r&   Tensor__annotations__r   r$   no_gradr   r=   rD   r/   r/   r-   r0   rE   I   s   
 
rE   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..Nr2   r1   r]   )r?   r&   rd   )rg   x1x2r/   r/   r0   rotate_halfm   s   rs   c                 C   sD   | |}| |}| | t| |  }|| t||  }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezers   )qkre   rf   rh   unsqueeze_dimq_embedk_embedr/   r/   r0   apply_rotary_pos_embt   s
   

rz   c                       s$   e Zd Z fddZdd Z  ZS )LlamaMLPc                    sx   t    || _|j| _|j| _tj| j| j|jd| _tj| j| j|jd| _	tj| j| j|jd| _
t|j | _d S )Nbias)r#   r$   rG   r+   intermediate_sizer   Linearmlp_bias	gate_projup_proj	down_projr   
hidden_actact_fnr*   rG   r-   r/   r0   r$      s   
zLlamaMLP.__init__c                 C   s$   |  | | || | }|S rm   )r   r   r   r   )r*   rg   r   r/   r/   r0   r=      s    zLlamaMLP.forward)rA   rB   rC   r$   r=   rD   r/   r/   r-   r0   r{      s    
r{   r:   n_repreturnc                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r?   r`   reshape)r:   r   batchnum_key_value_headsslenhead_dimr/   r/   r0   	repeat_kv   s
   0r           modulequerykeyvalueattention_maskscalingdropoutkwargsc                 K   s   t || j}t || j}	t||dd| }
|d ur3|d d d d d d d |jd f }|
| }
tjj|
dtj	d
|j}
tjj|
|| jd}
t|
|	}|dd }||
fS )Nr1   r   r2   )r^   r4   )ptrainingr   )r   num_key_value_groupsr&   matmulrc   r?   r   
functionalsoftmaxr6   r5   r4   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightscausal_maskattn_outputr/   r/   r0   eager_attention_forward   s   
&r   c                       s   e Zd ZdZdedef fddZedddd		
	
ddej	de
ej	ej	f deej	 dee deej dee de
ej	ej	f fddZ  ZS )LlamaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrG   	layer_idxc                    s   t    || _|| _t|d|j|j | _|j|j | _	| jd | _
|j| _d| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j| j |j|jd| _d S )Nr   g      Tr|   )r#   r$   rG   r   getattrr+   num_attention_headsr   r   r   r   attention_dropout	is_causalr   r   attention_biasq_projk_projv_projo_projr*   rG   r   r-   r/   r0   r$      s(   
zLlamaAttention.__init__past_key_valuepast_key_values4.58new_nameversionNr:   position_embeddingsr   cache_positionr   r   c                 K   s$  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}t|	|
||\}	}
|d urW|||d}||
|| j	|\}
}t
}| jjdkret| jj }|| |	|
||f| jsqdn| j| jd|\}}|jg |dR   }| |}||fS )Nr2   r   r1   )rf   re   r   eagerr   )r   r   )r?   r   r   viewrc   r   r   rz   updater   r   rG   _attn_implementationr   r   r   r   r   r   r   )r*   r:   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   re   rf   cache_kwargsattention_interfacer   r   r/   r/   r0   r=      s8   


zLlamaAttention.forward)NN)rA   rB   rC   __doc__r   intr$   r   r&   rn   r>   r   r   
LongTensorr   r   r=   rD   r/   r/   r-   r0   r      s*    r   c                       s   e Zd Zdedef fddZedddd							
				ddejde	ej de	ej
 de	e de	e de	ej
 de	eejejf  dee dejfddZ  ZS )LlamaDecoderLayerrG   r   c                    sR   t    |j| _t||d| _t|| _t|j|jd| _	t|j|jd| _
d S )N)rG   r   r,   )r#   r$   r+   r   	self_attnr{   mlpr!   rms_norm_epsinput_layernormpost_attention_layernormr   r-   r/   r0   r$     s   

zLlamaDecoderLayer.__init__r   r   r   r   NFr:   r   rh   	use_cacher   r   r   r   c              
   K   s^   |}	|  |}| jd|||||||d|\}}
|	| }|}	| |}| |}|	| }|S )N)r:   r   rh   r   r   r   r   r/   )r   r   r   r   )r*   r:   r   rh   r   r   r   r   r   residual_r/   r/   r0   r=     s&   




zLlamaDecoderLayer.forward)NNNFNN)rA   rB   rC   r   r   r$   r   r&   rn   r   r   r   boolr>   r   r   r=   rD   r/   r/   r-   r0   r     s8    
	
r   c                   @   sH   e Zd ZU eed< dZdZdgZdgZdZ	dZ
dZdZdZeedZdS )LlamaPreTrainedModelrG   modelTr   r   )r:   
attentionsN)rA   rB   rC   r   ro   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr   r   _can_record_outputsr/   r/   r/   r0   r   :  s   
 
r   c                       s   e Zd Zdef fddZee							ddeej	 deej
 deej	 dee d	eej d
eej	 dee dee defddZ  ZS )
LlamaModelrG   c                    s   t     j| _ j| _t j j| j| _t	 fddt
 jD | _t j jd| _t d| _d| _|   d S )Nc                    s   g | ]}t  |qS r/   )r   ).0r   rG   r/   r0   
<listcomp>V  s    z'LlamaModel.__init__.<locals>.<listcomp>r   r   F)r#   r$   pad_token_idpadding_idx
vocab_sizer   	Embeddingr+   embed_tokens
ModuleListrangenum_hidden_layerslayersr!   r   normrE   
rotary_embgradient_checkpointing	post_initr   r-   r   r0   r$   O  s   zLlamaModel.__init__N	input_idsr   rh   r   inputs_embedsr   r   r   r   c              	   K   s   |d u |d uA rt d|d u r| |}|r!|d u r!t| jd}|d u r=|d ur-| nd}	tj|	|	|jd  |jd}|d u rF|	d}t
| j|||||d}
|}| ||}| jd | jj D ]}||f|
||||d|}qb| |}t||dS )	Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r   )rX   )rG   input_embedsr   r   r   rh   )r   rh   r   r   r   )last_hidden_stater   )
ValueErrorr   r	   rG   get_seq_lengthr&   aranger?   rX   rt   r   r   r   r   r   r   )r*   r   r   rh   r   r   r   r   r   past_seen_tokensr   r:   r   decoder_layerr/   r/   r0   r=   _  sP   

	

zLlamaModel.forward)NNNNNNN)rA   rB   rC   r   r$   r   r   r   r&   r   rn   r   FloatTensorr   r   r   r   r=   rD   r/   r/   r-   r0   r   M  s<    	
r   c                       s   e Zd ZdgZddiZddgdgfiZ fddZee										dd
e	e
j de	e
j de	e
j de	e de	e
j de	e
j de	e de	e
j deee
jf dee defddZ  ZS )LlamaForCausalLMzlm_head.weightlm_headcolwise_repr:   logitsc                    s@   t  | t|| _|j| _tj|j|jdd| _| 	  d S )NFr|   )
r#   r$   r   r   r   r   r   r+   r   r   r   r-   r/   r0   r$     s
   
zLlamaForCausalLM.__init__Nr   r   r   rh   r   r   labelsr   r   logits_to_keepr   r   c
              
   K   s   | j d|||||||d|
}|j}t|	trt|	 dn|	}| |dd|ddf }d}|durB| jd||| jjd|
}t	|||j
|j|jdS )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, LlamaForCausalLM

        >>> model = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r   r   rh   r   r   r   r   N)r   r   r   )lossr   r   r:   r   r/   )r   r   rN   r   slicer   loss_functionrG   r   r   r   r:   r   )r*   r   r   rh   r   r   r   r   r   r   r   outputsr:   slice_indicesr   r   r/   r/   r0   r=     s0    zLlamaForCausalLM.forward)	NNNNNNNNr   )rA   rB   rC   _tied_weights_keys_tp_plan_pp_planr$   r   r   r   r&   r   rn   r   r   r   r   r   r   r   r   r=   rD   r/   r/   r-   r0   r     sN    		
r   c                   @      e Zd ZdS )LlamaForSequenceClassificationNrA   rB   rC   r/   r/   r/   r0   r        r  c                   @   s   e Zd ZdZdS )LlamaForQuestionAnsweringtransformerN)rA   rB   rC   r   r/   r/   r/   r0   r    s    r  c                   @   r  )LlamaForTokenClassificationNr	  r/   r/   r/   r0   r    r
  r  )r   r   r   r  r  r  )Nr   )r   )Ctypingr   r   r   r&   r   activationsr   cache_utilsr   r	   
generationr
   integrationsr   masking_utilsr   modeling_layersr   r   r   r   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.deprecationr   utils.genericr   configuration_llamar   
get_loggerrA   loggerModuler!   rE   rs   rz   r{   rn   r   r   r_   r   r   r   r   r   r   r  r  r  __all__r/   r/   r/   r0   <module>   sn   
$

G.NK