o
    ik                     @   sD  d dl mZmZmZ d dlZd dlm  mZ d dlmZ ddl	m
Z
 ddlmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z- edG dd dej.Z/G dd dej.Z0dd Z1d?ddZ2dej3de4d ej3fd!d"Z5	#d@d$ej.d%ej3d&ej3d'ej3d(eej3 d)e6d*e6d+e#e% fd,d-Z7G d.d/ d/ej.Z8G d0d1 d1ej.Z9G d2d3 d3ej.Z:G d4d5 d5ej.Z;G d6d7 d7eZ<e&G d8d9 d9e!Z=e&G d:d; d;e=Z>e&G d<d= d=e=eZ?g d>Z@dS )A    )CallableOptionalUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)deprecate_kwarg)check_model_inputs   )Dots1ConfigRMSNormc                       sF   e Zd Zddeddf fddZdejdejfdd	Zd
d Z  Z	S )Dots1RMSNormư>epsreturnNc                    s&   t    tt|| _|| _dS )z;
        Dots1RMSNorm is equivalent to T5LayerNorm
        N)super__init__r   	Parametertorchonesweightvariance_epsilon)selfhidden_sizer!   	__class__ e/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/transformers/models/dots1/modeling_dots1.pyr$   .   s   

zDots1RMSNorm.__init__hidden_statesc                 C   sJ   |j }|tj}|djddd}|t|| j  }| j|| S )N   T)keepdim)	dtypetor&   float32powmeanrsqrtr)   r(   )r*   r0   input_dtypevariancer.   r.   r/   forward6   s
   zDots1RMSNorm.forwardc                 C   s   t | jj d| j S )Nz, eps=)tupler(   shaper)   )r*   r.   r.   r/   
extra_repr=   s   zDots1RMSNorm.extra_repr)r    )
__name__
__module____qualname__floatr$   r&   Tensorr<   r?   __classcell__r.   r.   r,   r/   r   ,   s    r   c                       sD   e Zd ZU ejed< ddef fddZe e	dd Z
  ZS )	Dots1RotaryEmbeddinginv_freqNconfigc                    s   t    t|drt|jtr|jd|jd| _nd| _|j| _	|j| _
|| _t| j | _| | j|\}| _| jd|dd | j| _d S )Nrope_scaling	rope_typetypedefaultrG   F)
persistent)r#   r$   hasattr
isinstancerI   dictgetrJ   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrH   r   rope_init_fnattention_scalingregister_bufferrG   original_inv_freq)r*   rH   devicerG   r,   r.   r/   r$   D   s   
zDots1RotaryEmbedding.__init__c           
      C   s   | j d d d d f  |jd dd|j}|d d d d d f  }t|jjtr6|jjdkr6|jjnd}t	j
|dd+ | |  dd}t	j||fdd	}| | j }| | j }	W d    n1 smw   Y  |j|jd
|	j|jd
fS )Nr   r2   r   mpscpuF)device_typeenabledr1   dimr4   )rG   rC   expandr>   r5   rY   rO   rK   strr&   autocast	transposecatcosrV   sinr4   )
r*   xposition_idsinv_freq_expandedposition_ids_expandedr\   freqsembrf   rg   r.   r.   r/   r<   U   s   0&zDots1RotaryEmbedding.forwardN)r@   rA   rB   r&   rD   __annotations__r   r$   no_gradr   r<   rE   r.   r.   r,   r/   rF   A   s   
 
rF   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..Nr2   r1   r^   )r>   r&   re   )rh   x1x2r.   r.   r/   rotate_halfe   s   rs   c                 C   sD   | |}| |}| | t| |  }|| t||  }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezers   )qkrf   rg   ri   unsqueeze_dimq_embedk_embedr.   r.   r/   apply_rotary_pos_embl   s
   

rz   r0   n_repr"   c                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r>   ra   reshape)r0   r{   batchnum_key_value_headsslenhead_dimr.   r.   r/   	repeat_kv   s
   0r           modulequerykeyvalueattention_maskscalingdropoutkwargsc                 K   s   t || j}t || j}	t||dd| }
|d ur3|d d d d d d d |jd f }|
| }
tjj|
dtj	d
|j}
tjj|
|| jd}
t|
|	}|dd }||
fS )Nr1   r   r2   )r_   r4   )ptrainingr   )r   num_key_value_groupsr&   matmulrd   r>   r   
functionalsoftmaxr6   r5   r4   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightscausal_maskattn_outputr.   r.   r/   eager_attention_forward   s   
&r   c                       s   e Zd ZdZdedef fddZedddd		
	
ddej	de
ej	ej	f deej	 dee deej dee de
ej	eej	 f fddZ  ZS )Dots1Attentionz=Multi-headed attention from 'Attention Is All You Need' paperrH   	layer_idxc                    s  t    || _|| _t|d|j|j | _|j|j | _	| jd | _
|j| _d| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j| j |j|jd| _t| j|jd| _t| j|jd| _|j| dkr|j| _d S d | _d S )Nr   g      Tbiasr!   sliding_attention)r#   r$   rH   r   getattrr+   num_attention_headsr   r~   r   r   attention_dropout	is_causalr   Linearattention_biasq_projk_projv_projo_projr   rms_norm_epsq_normk_normlayer_typessliding_windowr*   rH   r   r,   r.   r/   r$      s.   
$zDots1Attention.__init__past_key_valuepast_key_values4.58new_nameversionNr0   position_embeddingsr   cache_positionr   r"   c                 K   s4  |j d d }g |d| jR }| | ||dd}	| | ||dd}
| ||dd}|\}}t	|	|
||\}	}
|d ur]|||d}|
|
|| j|\}
}t}| jjdkrkt| jj }|| |	|
||f| jswdn| j| j| jd|\}}|jg |dR   }| |}||fS )Nr2   r   r1   )rg   rf   r   eagerr   )r   r   r   )r>   r   r   r   viewrd   r   r   r   rz   updater   r   rH   _attn_implementationr   r   r   r   r   r|   r   r   )r*   r0   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   rf   rg   cache_kwargsattention_interfacer   r   r.   r.   r/   r<      s:   
	

zDots1Attention.forwardNN)r@   rA   rB   __doc__r   intr$   r   r&   rD   r=   r   r   
LongTensorr   r   r<   rE   r.   r.   r,   r/   r      s*    r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )Dots1MLPNc                    s   t    || _|d u r|jn|| _|d u r|jn|| _tj| j| jdd| _tj| j| jdd| _tj| j| jdd| _	t
|j | _d S NFr   )r#   r$   rH   r+   intermediate_sizer   r   	gate_projup_proj	down_projr   
hidden_actact_fn)r*   rH   r+   r   r,   r.   r/   r$      s   
zDots1MLP.__init__c                 C   s$   |  | | || | }|S rn   )r   r   r   r   )r*   rh   r   r.   r.   r/   r<     s    zDots1MLP.forwardr   )r@   rA   rB   r$   r<   rE   r.   r.   r,   r/   r      s    r   c                       sD   e Zd ZdZ fddZdejdejdejfddZd	d
 Z  Z	S )Dots1MoEz:
    A mixed expert module containing shared experts.
    c                    sT   t     | _t fddt jD | _t | _	t
  j j d| _d S )Nc                    s   g | ]	}t   jd qS ))r   )r   moe_intermediate_size).0_rH   r.   r/   
<listcomp>  s    z%Dots1MoE.__init__.<locals>.<listcomp>)rH   r   )r#   r$   rH   r   
ModuleListrangen_routed_expertsexpertsDots1TopkRoutergater   r   n_shared_expertsshared_expertsr*   rH   r,   r   r/   r$     s   

zDots1MoE.__init__r0   topk_indicestopk_weightsc                 C   s   t j||jd}t jjj|t| jd}|ddd}t	t| jD ]4}| j| }|| }t 
|\}	}
|	 dkrV||	|
f }||	 }||}||d }|d|	| q"||jS )z
        CALL FOR CONTRIBUTION! I don't have time to optimise this right now, but expert weights need to be fused
        to not have to do a loop here (deepseek has 256 experts soooo yeah).
        r`   )num_classesr1   r   r   r2   )r&   
zeros_liker4   r   r   one_hotlenr   permuter   wherenumelrt   
index_add_rK   )r*   r0   r   r   final_hidden_statesexpert_mask
expert_idxexpertmasktoken_indicesweight_indicesexpert_weightsexpert_inputexpert_outputweighted_outputr.   r.   r/   moe  s   
zDots1MoE.moec                 C   sP   |}|j }| |\}}|d|j d }| |||j| }|| | }|S )Nr2   )r>   r   r   r   r   )r*   r0   	residuals
orig_shaper   r   r.   r.   r/   r<   3  s   zDots1MoE.forward)
r@   rA   rB   r   r$   r&   rD   r   r<   rE   r.   r.   r,   r/   r   	  s
    r   c                       s4   e Zd Z fddZe dd Zdd Z  ZS )r   c                    sr   t    || _|j| _|j| _|j| _|j| _|j| _|j	| _	t
t| j|jf| _| dt| j d S )Ne_score_correction_bias)r#   r$   rH   num_experts_per_toktop_kr   routed_scaling_factorn_group
topk_groupnorm_topk_probr   r%   r&   emptyr+   r(   rW   zerosr   r,   r.   r/   r$   >  s   
zDots1TopkRouter.__init__c                 C   s   | d| j| jd }| d| j| j| j jdddd jdd}tj|| jdddd }t	|}|
d|d |dd| j| j| j d| j}||  d}tj|| jdddd }|S )	Nr2   r   r1   r^   F)rv   r_   sortedr   r   )r   r   r   rt   r   topksumr&   r   r   scatter_ra   r|   masked_fillboolr   )r*   scoresscores_for_choicegroup_scores	group_idx
group_mask
score_maskr   r.   r.   r/   get_topk_indicesK  s&   

z Dots1TopkRouter.get_topk_indicesc                 C   s~   | d| jj}t|tj| jtj}|	 }| 
|}|d|}| jr6|jdddd }|| }|| j }||fS )Nr2   r   T)r_   r3   g#B;)r   rH   r+   FlinearrK   r&   r6   r(   sigmoidr  gatherr   r   r   )r*   r0   router_logitsr  r   r   denominatorr.   r.   r/   r<   _  s   

zDots1TopkRouter.forward)	r@   rA   rB   r$   r&   rp   r  r<   rE   r.   r.   r,   r/   r   =  s
    
r   c                       s   e Zd Zdedef fddZedddd							
				ddejde	ej de	ej
 de	e de	e de	ej
 de	eejejf  dee dejfddZ  ZS )Dots1DecoderLayerrH   r   c                    st   t    |j| _t||d| _||jkrt|| _nt|| _t	|j|j
d| _t	|j|j
d| _|j| | _d S )N)rH   r   r   )r#   r$   r+   r   	self_attnfirst_k_dense_replacer   mlpr   r   r   input_layernormpost_attention_layernormr   attention_typer   r,   r.   r/   r$   m  s   


zDots1DecoderLayer.__init__r   r   r   r   NFr0   r   ri   	use_cacher   r   r   r"   c              
   K   s^   |}	|  |}| jd|||||||d|\}}
|	| }|}	| |}| |}|	| }|S )N)r0   r   ri   r   r  r   r   r.   )r  r  r  r  )r*   r0   r   ri   r   r  r   r   r   residualr   r.   r.   r/   r<   |  s&   




zDots1DecoderLayer.forward)NNNFNN)r@   rA   rB   r   r   r$   r   r&   rD   r   r   r   r  r=   r   r   r<   rE   r.   r.   r,   r/   r  l  s8    	
r  c                       sX   e Zd ZU eed< dZdZdgZdgZdZ	dZ
dZdZdZeedZ fdd	Z  ZS )
Dots1PreTrainedModelrH   modelTr  r   F)r0   
attentionsc                    s4   t  | t|tr|jjjd| jjd d S d S )Nr   )r8   std)	r#   _init_weightsrO   r   r(   datanormal_rH   initializer_range)r*   r   r,   r.   r/   r    s   
z"Dots1PreTrainedModel._init_weights)r@   rA   rB   r   ro   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr  r   _can_record_outputsr  rE   r.   r.   r,   r/   r    s   
 r  c                       s   e Zd Zdef fddZee							ddeej	 deej
 deej	 dee d	eej d
ee deej	 dee defddZ  ZS )
Dots1ModelrH   c                    s   t     j| _ j| _t j j| j| _t	 fddt
 jD | _t j jd| _t d| _d| _d| jjv | _|   d S )Nc                    s   g | ]}t  |qS r.   )r  )r   r   r   r.   r/   r     s    z'Dots1Model.__init__.<locals>.<listcomp>r   r   Fr   )r#   r$   pad_token_idpadding_idx
vocab_sizer   	Embeddingr+   embed_tokensr   r   num_hidden_layerslayersr   r   normrF   
rotary_embgradient_checkpointingrH   r   has_sliding_layers	post_initr   r,   r   r/   r$     s   zDots1Model.__init__N	input_idsr   ri   r   inputs_embedsr  r   r   r"   c              
   K   sF  |d u |d uA rt d|d u r| |}|r!|d u r!t| jd}|d u r=|d ur-| nd}	tj|	|	|jd  |jd}|d u rF|	d}t
| }
tsl| j|||||d}dtdi |i}
| jrltdi ||
d< |}| ||}| jd | jj D ]}||f|
|j |||||d	|}q}| |}t||r|d
S d d
S )Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r   )rY   )rH   input_embedsr   r   r   ri   full_attentionr   )r   ri   r   r  r   r   )last_hidden_stater   r.   )
ValueErrorr/  r	   rH   get_seq_lengthr&   aranger>   rY   rt   rO   rP   r   r5  r   r3  r1  r0  r  r2  r   )r*   r7  r   ri   r   r8  r  r   r   past_seen_tokenscausal_mask_mappingmask_kwargsr0   r   decoder_layerr.   r.   r/   r<     s^   



zDots1Model.forward)NNNNNNN)r@   rA   rB   r   r$   r   r   r   r&   r   rD   r   FloatTensorr  r   r   r   r<   rE   r.   r.   r,   r/   r*    s<    	
r*  c                       s   e Zd ZdgZddiZddgdgfiZ fddZee										dd
e	e
j de	e
j de	e
j de	e de	e
j de	e
j de	e de	e
j deee
jf dee defddZ  ZS )Dots1ForCausalLMzlm_head.weightlm_headcolwise_repr0   logitsc                    s@   t  | t|| _|j| _tj|j|jdd| _| 	  d S r   )
r#   r$   r*  r  r-  r   r   r+   rE  r6  r   r,   r.   r/   r$     s
   
zDots1ForCausalLM.__init__Nr   r7  r   ri   r   r8  labelsr  r   logits_to_keepr   r"   c
              
   K   s   | j d|||||||d|
}|j}t|	trt|	 dn|	}| |dd|ddf }d}|durB| jd||| jjd|
}t	|||j
|j|jdS )a~  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Dots1ForCausalLM

        >>> model = Dots1ForCausalLM.from_pretrained("rednote-hilab/dots1.llm1.inst")
        >>> tokenizer = AutoTokenizer.from_pretrained("rednote-hilab/dots1.llm1.inst")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r7  r   ri   r   r8  r  r   N)rG  rH  r-  )lossrG  r   r0   r  r.   )r  r;  rO   r   slicerE  loss_functionrH   r-  r   r   r0   r  )r*   r7  r   ri   r   r8  rH  r  r   rI  r   outputsr0   slice_indicesrG  rJ  r.   r.   r/   r<   "  s0   %zDots1ForCausalLM.forward)	NNNNNNNNr   )r@   rA   rB   _tied_weights_keys_tp_plan_pp_planr$   r   r   r   r&   r   rD   r   rC  r  r   r   r   r   r   r<   rE   r.   r.   r,   r/   rD    sN    		
rD  )r  r*  rD  )Nr   )r   )Atypingr   r   r   r&   torch.nn.functionalr   r   r	  activationsr   cache_utilsr   r	   
generationr
   integrationsr   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.deprecationr   utils.genericr   configuration_dots1r   Moduler   rF   rs   rz   rD   r   r   rC   r   r   r   r   r   r  r  r*  rD  __all__r.   r.   r.   r/   <module>   sn   $

K4/3\P