o
    ۷iR                     @   s  d dl mZmZmZ d dlZd dlmZ ddlmZ ddlm	Z	m
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( G dd dej)Z*edG dd dej)Z+G dd dej)Z,dd Z-d<ddZ.dej/d e0d!ej/fd"d#Z1	$d=d%ej)d&ej/d'ej/d(ej/d)eej/ d*e2d+e2d,ee  fd-d.Z3G d/d0 d0ej)Z4G d1d2 d2eZ5e!G d3d4 d4eZ6e!G d5d6 d6e6Z7e!G d7d8 d8e6eZ8G d9d: d:ee6Z9g d;Z:dS )>    )CallableOptionalUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask)GenericForTokenClassificationGradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)deprecate_kwarg)check_model_inputs   )ApertusConfigc                       s$   e Zd Z fddZdd Z  ZS )
ApertusMLPc                    s\   t    || _|j| _|j| _tj| j| jdd| _tj| j| jdd| _t	|j
 | _d S NFbias)super__init__confighidden_sizeintermediate_sizer   Linearup_proj	down_projr   
hidden_actact_fnselfr#   	__class__ b/home/ubuntu/vllm_env/lib/python3.10/site-packages/transformers/models/apertus/modeling_apertus.pyr"   ,   s   
zApertusMLP.__init__c                 C   s   |  | | |S N)r(   r*   r'   )r,   xr/   r/   r0   forward5   s   zApertusMLP.forward)__name__
__module____qualname__r"   r3   __classcell__r/   r/   r-   r0   r   +   s    	r   RMSNormc                       s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	ApertusRMSNormư>c                    s&   t    tt|| _|| _dS )z=
        ApertusRMSNorm is equivalent to T5LayerNorm
        N)r!   r"   r   	Parametertorchonesweightvariance_epsilon)r,   r$   epsr-   r/   r0   r"   ;   s   

zApertusRMSNorm.__init__c                 C   sJ   |j }|tj}|djddd}|t|| j  }| j|| S )N   T)keepdim)	dtypetor<   float32powmeanrsqrtr?   r>   )r,   hidden_statesinput_dtypevariancer/   r/   r0   r3   C   s
   zApertusRMSNorm.forwardc                 C   s   t | jj d| j S )Nz, eps=)tupler>   shaper?   )r,   r/   r/   r0   
extra_reprJ   s   zApertusRMSNorm.extra_repr)r:   )r4   r5   r6   r"   r3   rO   r7   r/   r/   r-   r0   r9   9   s    r9   c                       sD   e Zd ZU ejed< ddef fddZe e	dd Z
  ZS )	ApertusRotaryEmbeddinginv_freqNr#   c                    s   t    t|drt|jtr|jd|jd| _nd| _|j| _	|j| _
|| _t| j | _| | j|\}| _| jd|dd | j| _d S )Nrope_scaling	rope_typetypedefaultrQ   F)
persistent)r!   r"   hasattr
isinstancerR   dictgetrS   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr#   r   rope_init_fnattention_scalingregister_bufferrQ   original_inv_freq)r,   r#   devicerQ   r-   r/   r0   r"   Q   s   
zApertusRotaryEmbedding.__init__c           
      C   s   | j d d d d f  |jd dd|j}|d d d d d f  }t|jjtr6|jjdkr6|jjnd}t	j
|dd+ | |  dd}t	j||fdd	}| | j }| | j }	W d    n1 smw   Y  |j|jd
|	j|jd
fS )Nr   rB   r   mpscpuF)device_typeenabledrA   dim)rD   )rQ   floatexpandrN   rE   rb   rX   rT   strr<   autocast	transposecatcosr_   sinrD   )
r,   r2   position_idsinv_freq_expandedposition_ids_expandedre   freqsembro   rp   r/   r/   r0   r3   b   s   0&zApertusRotaryEmbedding.forwardr1   )r4   r5   r6   r<   Tensor__annotations__r   r"   no_gradr   r3   r7   r/   r/   r-   r0   rP   N   s   
 
rP   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..NrB   rA   rg   )rN   r<   rn   )r2   x1x2r/   r/   r0   rotate_halfr   s   r{   c                 C   sD   | |}| |}| | t| |  }|| t||  }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer{   )qkro   rp   rq   unsqueeze_dimq_embedk_embedr/   r/   r0   apply_rotary_pos_emby   s
   

r   rJ   n_repreturnc                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rN   rj   reshape)rJ   r   batchnum_key_value_headsslenhead_dimr/   r/   r0   	repeat_kv   s
   0r           modulequerykeyvalueattention_maskscalingdropoutkwargsc                 K   s   t || j}t || j}	t||dd| }
|d ur3|d d d d d d d |jd f }|
| }
tjj|
dtj	d
|j}
tjj|
|| jd}
t|
|	}|dd }||
fS )NrA   r   rB   )rh   rD   )ptrainingr   )r   num_key_value_groupsr<   matmulrm   rN   r   
functionalsoftmaxrF   rE   rD   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightscausal_maskattn_outputr/   r/   r0   eager_attention_forward   s   
&r   c                       s   e Zd ZdZddedee f fddZeddd	d
		dde	j
dee	j
e	j
f dee	j
 dee dee	j dee dee	j
e	j
f fddZ  ZS )ApertusAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNr#   	layer_idxc                    s   t    || _|| _t|d|j|j | _|j|j | _	| jd | _
|j| _d| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j| j |j|jd| _t| j|j| _t| j|j| _d S )Nr   g      Tr   )r!   r"   r#   r   getattrr$   num_attention_headsr   r   r   r   attention_dropout	is_causalr   r&   attention_biasq_projk_projv_projo_projr9   rms_norm_epsq_normk_normr,   r#   r   r-   r/   r0   r"      s,   
zApertusAttention.__init__past_key_valuepast_key_values4.58new_nameversionrJ   position_embeddingsr   cache_positionr   r   c                 K   s8  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}| |	}	| |
}
|\}}t	|	|
||\}	}
|d ura|||d}|
|
|| j|\}
}t}| jjdkrot| jj }|| |	|
||f| js{dn| j| jd|\}}|jg |dR   }| |}||fS )NrB   r   rA   )rp   ro   r   eagerr   )r   r   )rN   r   r   viewrm   r   r   r   r   r   updater   r   r#   _attn_implementationr   r   r   r   r   r   r   )r,   rJ   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   ro   rp   cache_kwargsattention_interfacer   r   r/   r/   r0   r3      s<   




zApertusAttention.forwardr1   )NN)r4   r5   r6   __doc__r   r   intr"   r   r<   rv   rM   r   
LongTensorr   r   r3   r7   r/   r/   r-   r0   r      s*    r   c                       s   e Zd Zdedef fddZedddd							
				ddejde	ej de	ej
 de	e de	e de	ej
 de	eejejf  dee deej fddZ  ZS )ApertusDecoderLayerr#   r   c                    sR   t    |j| _t||d| _t|| _t|j|jd| _	t|j|jd| _
d S )N)r#   r   r@   )r!   r"   r$   r   	self_attnr   mlpr9   r   attention_layernormfeedforward_layernormr   r-   r/   r0   r"     s   

zApertusDecoderLayer.__init__r   r   r   r   NFrJ   r   rq   	use_cacher   r   r   r   c              
   K   s^   |}	|  |}| jd|||||||d|\}}
|	| }|}	| |}| |}|	| }|S )N)rJ   r   rq   r   r   r   r   r/   )r   r   r   r   )r,   rJ   r   rq   r   r   r   r   r   residual_r/   r/   r0   r3     s&   




zApertusDecoderLayer.forward)NNNFNN)r4   r5   r6   r   r   r"   r   r<   rv   r   r   r   boolrM   r   r   r3   r7   r/   r/   r-   r0   r     s8    
	
r   c                   @   sH   e Zd ZU eed< dZdZdgZdgZdZ	dZ
dZdZdZeedZdS )ApertusPreTrainedModelr#   modelTr   r   )rJ   
attentionsN)r4   r5   r6   r   rw   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr   r   _can_record_outputsr/   r/   r/   r0   r   1  s   
 
r   c                       s   e Zd Zdef fddZe e							ddeej	 deej
 deej	 dee d	eej d
eej	 dee dee defddZ  ZS )ApertusModelr#   c                    s   t     j| _ j| _t j j| j| _t	 fddt
 jD | _t j jd| _t d| _d| _|   d S )Nc                    s   g | ]}t  |qS r/   )r   ).0r   r#   r/   r0   
<listcomp>M  s    z)ApertusModel.__init__.<locals>.<listcomp>r   r   F)r!   r"   pad_token_idpadding_idx
vocab_sizer   	Embeddingr$   embed_tokens
ModuleListrangenum_hidden_layerslayersr9   r   normrP   
rotary_embgradient_checkpointing	post_initr+   r-   r   r0   r"   F  s   zApertusModel.__init__N	input_idsr   rq   r   inputs_embedsr   r   r   r   c              	   K   s   |d u |d uA rt d|d u r| |}|r!|d u r!t| jd}|d u r=|d ur-| nd}	tj|	|	|jd  |jd}|d u rF|	d}t
| j|||||d}
|}| ||}| jd | jj D ]}||f|
||||d|}qb| |}t||dS )	Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r   )rb   )r#   input_embedsr   r   r   rq   )r   rq   r   r   r   )last_hidden_stater   )
ValueErrorr   r	   r#   get_seq_lengthr<   arangerN   rb   r|   r   r   r   r   r   r   )r,   r   r   rq   r   r   r   r   r   past_seen_tokensr   rJ   r   decoder_layerr/   r/   r0   r3   V  sP   

	

zApertusModel.forward)NNNNNNN)r4   r5   r6   r   r"   r   r   r   r<   r   rv   r   FloatTensorr   r   r   r   r3   r7   r/   r/   r-   r0   r   D  s<    	
r   c                       s   e Zd ZdgZddiZddgdgfiZ fddZee										dd
e	e
j de	e
j de	e
j de	e de	e
j de	e
j de	e de	e
j deee
jf dee defddZ  ZS )ApertusForCausalLMzlm_head.weightlm_headcolwise_reprJ   logitsc                    s@   t  | t|| _|j| _tj|j|jdd| _| 	  d S r   )
r!   r"   r   r   r   r   r&   r$   r   r   r+   r-   r/   r0   r"     s
   
zApertusForCausalLM.__init__Nr   r   r   rq   r   r   labelsr   r   logits_to_keepr   r   c
              
   K   s   | j d|||||||d|
}|j}t|	trt|	 dn|	}| |dd|ddf }d}|durB| jd||| jjd|
}t	|||j
|j|jdS )an  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ApertusForCausalLM

        >>> model = ApertusForCausalLM.from_pretrained("swiss-ai/Apertus-8B")
        >>> tokenizer = AutoTokenizer.from_pretrained("swiss-ai/Apertus-8B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r   r   rq   r   r   r   r   N)r   r   r   )lossr   r   rJ   r   r/   )r   r   rX   r   slicer   loss_functionr#   r   r   r   rJ   r   )r,   r   r   rq   r   r   r   r   r   r   r   outputsrJ   slice_indicesr   r   r/   r/   r0   r3     s0   %zApertusForCausalLM.forward)	NNNNNNNNr   )r4   r5   r6   _tied_weights_keys_tp_plan_pp_planr"   r   r   r   r<   r   rv   r   r   r   r   r   r   r   r   r3   r7   r/   r/   r-   r0   r     sN    		
r   c                   @   s   e Zd ZdS )ApertusForTokenClassificationN)r4   r5   r6   r/   r/   r/   r0   r    s    r  )r   r   r  r   )Nr   )r   );typingr   r   r   r<   r   activationsr   cache_utilsr   r	   
generationr
   integrationsr   masking_utilsr   modeling_layersr   r   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.deprecationr   utils.genericr   configuration_apertusr   Moduler   r9   rP   r{   r   rv   r   r   ri   r   r   r   r   r   r   r  __all__r/   r/   r/   r0   <module>   sh   $

J-NP