o
    wiX                     @   s  d dl mZmZmZ d dlZd dlm  mZ d dlmZ ddl	m
Z
 ddlmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z, ddl-m.Z. e,/e0Z1edG dd dej2Z3G dd deZ4G dd dej2Z5dd Z6dMddZ7dej8de9d ej8fd!d"Z:	#dNd$ej2d%ej8d&ej8d'ej8d(eej8 d)e;d*e;fd+d,Z<G d-d. d.ej2Z=G d/d0 d0ej2Z>G d1d2 d2ej2Z?G d3d4 d4eZ@e*G d5d6 d6e%ZAG d7d8 d8ej2ZBe*G d9d: d:eAZCG d;d< d<ee)ZD		=	dOd>eej8eEej8 df d?ee9 d(eej8 d eej8e9f fd@dAZFe*G dBdC dCeAeZGe*dDdEG dFdG dGeAZHe*G dHdI dIeAZIe*G dJdK dKeAZJg dLZKdS )P    )CallableOptionalUnionN)nn   )ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastMoeCausalLMOutputWithPastMoeModelOutputWithPastQuestionAnsweringModelOutput SequenceClassifierOutputWithPastTokenClassifierOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)
LossKwargsauto_docstringcan_return_tuplelogging   )MiniMaxConfigRMSNormc                       s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	MiniMaxRMSNormư>c                    s&   t    tt|| _|| _dS )z=
        MiniMaxRMSNorm is equivalent to T5LayerNorm
        N)super__init__r   	Parametertorchonesweightvariance_epsilon)selfhidden_sizeeps	__class__ i/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/minimax/modeling_minimax.pyr%   8   s   

zMiniMaxRMSNorm.__init__c                 C   sJ   |j }|tj}|djddd}|t|| j  }| j|| S )N   T)keepdim)	dtypetor'   float32powmeanrsqrtr*   r)   )r+   hidden_statesinput_dtypevariancer0   r0   r1   forward@   s
   zMiniMaxRMSNorm.forwardc                 C   s   t | jj d| j S )Nz, eps=)tupler)   shaper*   r+   r0   r0   r1   
extra_reprG   s   zMiniMaxRMSNorm.extra_repr)r#   )__name__
__module____qualname__r%   r>   rB   __classcell__r0   r0   r.   r1   r"   6   s    r"   c                       s   e Zd Z fddZdd ZdefddZ fdd	Zdef fd
dZdd Z	defddZ
dejfddZdefddZ  ZS )MiniMaxCachec                    s   t    g | _d S N)r$   r%   linear_cacherA   r.   r0   r1   r%   L   s   

zMiniMaxCache.__init__c                 C   s4   t t| j|d D ]}| jg  q
|| j|< d S Nr   )rangelenrI   append)r+   	layer_idxrI   _r0   r0   r1   set_linear_cacheP   s   zMiniMaxCache.set_linear_cacherN   c                 C   s   |t | k r| j| S d S rH   )rL   rI   r+   rN   r0   r0   r1   get_linear_cacheV   s   
zMiniMaxCache.get_linear_cachec                    s   t t  t| jS rH   )maxr$   __len__rL   rI   rA   r.   r0   r1   rT   [   s   zMiniMaxCache.__len__c                    s4   |t | jk r| j| g kr| j| fS t |S rH   )rL   rI   r$   __getitem__rQ   r.   r0   r1   rU   ^   s   zMiniMaxCache.__getitem__c                 c   s"    t t| D ]}| | V  qd S rH   )rK   rL   rQ   r0   r0   r1   __iter__c   s   zMiniMaxCache.__iter__repeatsc                 C   st   t t| D ]1}| j| g kr| j| j|dd| j|< q| j| j|dd| j|< | j| j|dd| j|< qd S )Nr   dim)rK   rL   rI   repeat_interleave	key_cachevalue_cache)r+   rW   rN   r0   r0   r1   batch_repeat_interleaveg   s   z$MiniMaxCache.batch_repeat_interleaveindicesc                 C   sn   t t| D ].}| j| g kr| j| |df | j|< q| j| |df | j|< | j| |df | j|< qd S )N.)rK   rL   rI   r[   r\   )r+   r^   rN   r0   r0   r1   batch_select_indiceso   s   z!MiniMaxCache.batch_select_indices
max_lengthc                 C   s   t d)Nz*MiniMaxCache doesnot support `crop` method)RuntimeError)r+   r`   r0   r0   r1   cropw      zMiniMaxCache.crop)rC   rD   rE   r%   rP   intrR   rT   rU   rV   r]   r'   Tensorr_   rb   rF   r0   r0   r.   r1   rG   K   s    rG   c                       s   e Zd Zdedef fddZdd Zdd Z				dd
ej	de
ej	ej	f deej	 dee deej dee de
ej	eej	 ee
ej	  f fddZ  ZS )MiniMaxLightningAttentionconfigrN   c                    s  t    || _t|dd p|j|j | _|j| _|j| _|j| _t	|j
 | _t| j| j | _tj|j| j| j d dd| _tj| j| j |jdd| _tj|j| j| j dd| _|  }| |\}}}| d| | d| | d| | d| d S )	Nhead_dimr   Fbias
slope_ratequery_decay	key_decaydiagonal_decay)r$   r%   rN   getattrr,   num_attention_headsrh   num_hidden_layers
block_sizer   
hidden_actact_fnr"   normr   Linearqkv_projout_projoutput_gateget_slope_ratedecay_factorsregister_buffer)r+   rg   rN   rk   rl   rm   rn   r.   r0   r1   r%   |   s"   
 z"MiniMaxLightningAttention.__init__c                 C   sd   ddd| j    }t| j d }d| j| jd d   d }|| }|| }|d d d d f }|S )Nr   r2      gh㈵>)rp   r'   arangerN   rq   )r+   baseexponentfactorrater0   r0   r1   rz      s   z(MiniMaxLightningAttention.get_slope_ratec                 C   s   t | jd }t | |d d d f  }t | | j|d d d f   }|d d d f |d d d f  }|d d d d d d f }|| }t |dk| td}t |}|||fS )Nr   r   z-inf)r'   r~   rr   expwherefloat)r+   rk   block_size_rangerl   rm   rn   r0   r0   r1   r{      s   " 

z'MiniMaxLightningAttention.decay_factorsNr;   position_embeddingsattention_maskpast_key_valuecache_positionkwargsreturnc           #      K   sl  |j \}}}	|| j d | j }
| | |}|||| jd| j }tj|| jdd\}}}|	dd}|	dd}|	dd}d }|d urN|
| j}|d u r!t|| j| j| j|}|d ury|jtjd}||dd d}g }t|
D ]}|| j }t|| j |}|| }|d d d d ||f }|d d d d ||f }|d d d d ||f }| jd d d |f }| jd d | d f }| jd d d d d |d |f }t| j | }t||	dd}t|| |}t|| |}|| }|| t|| 	dd|} || |  }qnYt| j }!g }t|D ]K}|d d d d ||d f }|d d d d ||d f }|d d d d ||d f }t|	dd|}"|!| |" }t||}|| q.tj|dd}|	dd}|||| j| j }| |}t| || }| |}|d ur| | j| ||fS )	Nr   r   rX   r2   r5   r3   r   )!r@   rr   rt   rw   reshaperp   rh   r'   split	transposerR   rN   zerosr6   boolmasked_fill	unsqueezerK   minrl   rm   rn   r   rk   matmulrM   catru   Fsigmoidry   rx   rP   )#r+   r;   r   r   r   r   r   
batch_sizeseq_lenr,   
num_blocks
qkv_statesquery_states
key_statesvalue_statesattn_weights_interattn_outputi	start_idxend_idxcurrent_block_sizecurrent_query_statescurrent_key_statescurrent_value_statescurrent_query_decaycurrent_key_decaycurrent_diagonal_decayblock_decayattn_weights_intraattn_output_intraattn_output_intercurrent_attn_outputnext_attn_weights_interratiocurrent_attn_weights_interr0   r0   r1   r>      sv   	

"
 


z!MiniMaxLightningAttention.forwardNN)rC   rD   rE   r    rd   r%   rz   r{   r'   re   r?   r   r   
LongTensorr   r   r>   rF   r0   r0   r.   r1   rf   {   s*    rf   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..Nr3   r2   rX   )r@   r'   r   )xx1x2r0   r0   r1   rotate_half  s   r   c                 C   sD   | |}| |}| | t| |  }|| t||  }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r   r   )qkcossinposition_idsunsqueeze_dimq_embedk_embedr0   r0   r1   apply_rotary_pos_emb  s
   

r   r;   n_repr   c                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r@   expandr   )r;   r   batchnum_key_value_headsslenrh   r0   r0   r1   	repeat_kv0  s
   0r           modulequerykeyvaluer   scalingdropoutc                 K   s   t || j}t || j}	t||dd| }
|d ur3|d d d d d d d |jd f }|
| }
tjj|
dtj	d
|j}
tjj|
|| jd}
t|
|	}|dd }||
fS )Nr2   r   r   r3   rY   r5   )ptrainingr   )r   num_key_value_groupsr'   r   r   r@   r   
functionalsoftmaxr7   r6   r5   r   r   
contiguous)r   r   r   r   r   r   r   r   r   r   attn_weightscausal_maskr   r0   r0   r1   eager_attention_forward<  s   
&r   c                       s   e Zd ZdZdedef fddZ		ddejde	ejejf d	e
ej d
e
e de
ej dee de	eje
ej e
e	ej  f fddZ  ZS )MiniMaxAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrg   rN   c                    s   t    || _|| _t|dd p|j|j | _|j|j | _	| jd | _
|j| _d| _tj|j|j| j dd| _tj|j|j| j dd| _tj|j|j| j dd| _tj|j| j |jdd| _d S )Nrh   g      TFri   )r$   r%   rg   rN   ro   r,   rp   rh   r   r   r   attention_dropout	is_causalr   rv   q_projk_projv_projo_projr+   rg   rN   r.   r0   r1   r%   Y  s   
 zMiniMaxAttention.__init__Nr;   r   r   r   r   r   r   c                 K   s0  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}t|	|
||\}	}
|d urW|||d}||
|| j	|\}
}t
}| jjdkret| jj }|| |	|
||f| jsqdn| j| jt| jdd d|\}}|jg |dR   }| |}||fS )	Nr3   r   r2   )r   r   r   eagerr   sliding_window)r   r   r   )r@   rh   r   viewr   r   r   r   updaterN   r   rg   _attn_implementationr   r   r   r   ro   r   r   r   )r+   r;   r   r   r   r   r   input_shapehidden_shaper   r   r   r   r   cache_kwargsattention_interfacer   r   r0   r0   r1   r>   g  s:   		

zMiniMaxAttention.forwardr   )rC   rD   rE   __doc__r    rd   r%   r'   re   r?   r   r   r   r   r   r>   rF   r0   r0   r.   r1   r   V  s(    r   c                       s*   e Zd Zdef fddZdd Z  ZS )MiniMaxBlockSparseTop2MLPrg   c                    sl   t    |j| _|j| _tj| j| jdd| _tj| j| jdd| _	tj| j| jdd| _
t|j | _d S NFri   )r$   r%   intermediate_sizeffn_dimr,   
hidden_dimr   rv   w1w2w3r   rs   rt   r+   rg   r.   r0   r1   r%     s   
z"MiniMaxBlockSparseTop2MLP.__init__c                 C   s(   |  | || | }| |}|S rH   )rt   r   r   r   )r+   r;   current_hidden_statesr0   r0   r1   r>     s   
z!MiniMaxBlockSparseTop2MLP.forward)rC   rD   rE   r    r%   r>   rF   r0   r0   r.   r1   r     s    r   c                       s6   e Zd ZdZ fddZdejdejfddZ  ZS )MiniMaxSparseMoeBlocka  
    This implementation is
    strictly equivalent to standard MoE with full capacity (no
    dropped tokens). It's faster since it formulates MoE operations
    in terms of block-sparse operations to accommodate imbalanced
    assignments of tokens to experts, whereas standard MoE either
    (1) drop tokens at the cost of reduced performance or (2) set
    capacity factor to number of experts and thus waste computation
    and memory on padding.
    c                    sl   t     j| _ j| _ j| _ j| _	t
j| j| jdd| _t
 fddt| jD | _ j| _d S )NFri   c                    s   g | ]}t  qS r0   )r   ).0rO   rg   r0   r1   
<listcomp>  s    z2MiniMaxSparseMoeBlock.__init__.<locals>.<listcomp>)r$   r%   r,   r   r   r   num_local_expertsnum_expertsnum_experts_per_toktop_kr   rv   gate
ModuleListrK   expertsrouter_jitter_noisejitter_noiser   r.   r   r1   r%     s   
 zMiniMaxSparseMoeBlock.__init__r;   r   c                 C   sn  |j \}}}| jr| jdkr|t|d| j d| j 9 }|d|}| |}tj	|dtj
d}tj|| jdd\}}||jddd }||j}tj|| |f|j|jd	}tjjj|| jd
ddd}	t|	jddd }
|
D ]3}| j| }t|	| d\}}|d|f d|}|||||df  }|d|||j qx||||}||fS ) r         ?r3   r   r   rX   T)rY   r4   )r5   device)num_classesr2   )r3   r   N)r@   r   r  r'   
empty_likeuniform_r   r   r   r   r   topkr   sumr6   r5   r   r  r   r   one_hotr   permutegreaternonzeror  r   squeezer   
index_add_)r+   r;   r   sequence_lengthr   router_logitsrouting_weightsselected_expertsfinal_hidden_statesexpert_maskexpert_hitted
expert_idxexpert_layeridxtop_xcurrent_stater   r0   r0   r1   r>     s,   "

zMiniMaxSparseMoeBlock.forward)	rC   rD   rE   r   r%   r'   re   r>   rF   r0   r0   r.   r1   r     s    r   c                       s   e Zd Zdedef fddZ							ddejdeejejf d	e	ej d
e	ej
 de	eej  de	e de	e de	e de	ej
 dee deeje	eejejf  f fddZ  ZS )MiniMaxDecoderLayerrg   rN   c                    s   t    |j| _t||| _t|| _t|j|jd| _	t|j|jd| _
|| _|j| | _|j| _|j| _| jdkrLt||| _|j| _|j| _d S t||| _|j| _|j| _d S )Nr-   linear_attention)r$   r%   r,   r   	self_attnr   block_sparse_moer"   rms_norm_epsinput_layernormpost_attention_layernormrN   layer_types
layer_typemlp_alpha_factormlp_beta_factorrf   linear_attn_alpha_factorattn_alpha_factorlinear_attn_beta_factorattn_beta_factorfull_attn_alpha_factorfull_attn_beta_factorr   r.   r0   r1   r%     s"   


zMiniMaxDecoderLayer.__init__NFr;   r   r   r   r   output_attentionsoutput_router_logits	use_cacher   r   r   c
                 K   s   |  |}|}| jd||||||||	d|
\}}|| j || j  }| |}|}| |\}}|| j || j  }|f}|rF||f7 }|rM||f7 }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            attention_mask (`torch.Tensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_router_logits (`bool`, *optional*):
                Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
                should not be returned during inference.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        )r;   r   r   r   r   r1  r3  r   Nr0   )r%  r"  r,  r.  r&  r#  r)  r*  )r+   r;   r   r   r   r   r1  r2  r3  r   r   residualself_attn_weightsr  outputsr0   r0   r1   r>     s2   
&	



zMiniMaxDecoderLayer.forward)NNNFFFN)rC   rD   rE   r    rd   r%   r'   re   r?   r   r   r   r   r   FloatTensorr>   rF   r0   r0   r.   r1   r    s@    	
r  c                   @   sL   e Zd ZeZdZdZdgZdgZdZ	dZ
dZdZdZdZdZdZdd ZdS )	MiniMaxPreTrainedModelmodelTr  past_key_valuesFc                 C   s   | j j}t|tjr"|jjjd|d |jd ur |jj	  d S d S t|tj
rC|jjjd|d |jd urA|jj|j 	  d S d S t|trQ|jjd d S d S )Nr   )r9   stdr  )rg   initializer_range
isinstancer   rv   r)   datanormal_rj   zero_	Embeddingpadding_idxr"   fill_)r+   r   r;  r0   r0   r1   _init_weightsZ  s   


z$MiniMaxPreTrainedModel._init_weightsN)rC   rD   rE   r    config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_3_supports_flash_attn_2_supports_sdpa_supports_flex_attn_supports_cache_class_supports_quantized_cache_supports_static_cache_supports_attention_backendrD  r0   r0   r0   r1   r8  J  s    r8  c                       s8   e Zd Zddef fddZe edd Z  Z	S )MiniMaxRotaryEmbeddingNrg   c                    s   t    t|dr|jd ur|jd|jd| _nd| _|j| _|j| _|| _	t
| j | _| | j	|\}| _| jd|dd | j| _d S )Nrope_scaling	rope_typetypedefaultinv_freqF)
persistent)r$   r%   hasattrrS  getrT  max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrg   r   rope_init_fnattention_scalingr|   rW  original_inv_freq)r+   rg   r  rW  r.   r0   r1   r%   i  s   
zMiniMaxRotaryEmbedding.__init__c           
      C   s   | j d d d d f  |jd dd|j}|d d d d d f  }t|jjtr6|jjdkr6|jjnd}t	j
|dd+ | |  dd}t	j||fdd	}| | j }| | j }	W d    n1 smw   Y  |j|jd
|	j|jd
fS )Nr   r3   r   mpscpuF)device_typeenabledr2   rX   r   )rW  r   r   r@   r6   r  r=  rU  strr'   autocastr   r   r   r_  r   r5   )
r+   r   r   inv_freq_expandedposition_ids_expandedrc  freqsembr   r   r0   r0   r1   r>   z  s   0&zMiniMaxRotaryEmbedding.forwardrH   )
rC   rD   rE   r    r%   r'   no_gradr   r>   rF   r0   r0   r.   r1   rR  h  s
    rR  c                       s   e Zd Zdef fddZdd Zdd Zee										dd	e	j
d
ee	j dee	j
 deee	j  dee	j dee dee dee dee dee	j
 dee defddZ  ZS )MiniMaxModelrg   c                    s   t     j| _ j| _t j j| j| _t	 fddt
 jD | _t j jd| _t d| _d| _|   d S )Nc                    s   g | ]}t  |qS r0   )r  )r   rN   r   r0   r1   r         z)MiniMaxModel.__init__.<locals>.<listcomp>r   r   F)r$   r%   pad_token_idrB  
vocab_sizer   rA  r,   embed_tokensr  rK   rq   layersr"   r$  ru   rR  
rotary_embgradient_checkpointing	post_initr   r.   r   r1   r%     s   zMiniMaxModel.__init__c                 C      | j S rH   rp  rA   r0   r0   r1   get_input_embeddings     z!MiniMaxModel.get_input_embeddingsc                 C   
   || _ d S rH   rv  r+   r   r0   r0   r1   set_input_embeddings     
z!MiniMaxModel.set_input_embeddingsN	input_idsr   r   r:  inputs_embedsr3  r1  output_hidden_statesr2  r   flash_attn_kwargsr   c                 K   s  |d ur|n| j j}|	d ur|	n| j j}	|d ur|n| j j}|d ur$|n| j j}|d u |d uA r4td| jrC| jrC|rCt	d d}|rM|d u rMt
 }n|r^t|t
s^tdt| d|d u rg| |}|
d u r|d urs| nd}tj|||jd  |jd}
|d u r|
d}| j jd u rtnt}|| j |||
||d	}|}| ||}|rd
nd }|rd
nd }|	rd
nd }| jD ]:}|r||f7 }|jdkr|}n|}||f||||||	||
d|}|d }|r||d f7 }|	r||d f7 }q| |}|r||f7 }t|||||dS )Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FzSMiniMax uses cache of its own and is not compatible with `past_key_values` of type .r   r   r  )rg   input_embedsr   r   r:  r   r0   full_attention)r   r   r   r   r1  r2  r3  r   r3   )last_hidden_stater:  r;   
attentionsr  )rg   r1  r2  r  r3  
ValueErrorrs  r   loggerwarning_oncerG   r=  rU  rp  get_seq_lengthr'   r~   r@   r  r   r   r   r   rr  rq  r(  ru   r   )r+   r}  r   r   r:  r~  r3  r1  r  r2  r   r  past_seen_tokensmask_functionr   r;   r   all_hidden_statesall_self_attnsall_router_logitsdecoder_layerinput_attention_masklayer_outputsr0   r0   r1   r>     s   

	





zMiniMaxModel.forward)
NNNNNNNNNN)rC   rD   rE   r    r%   rw  r{  r   r   r'   r   r   re   listr7  r   r   r   r   r>   rF   r0   r0   r.   r1   rl    sR    	
rl  c                   @   s   e Zd ZdS )KwargsForCausalLMN)rC   rD   rE   r0   r0   r0   r1   r    s    r  r2   gate_logitsr   c                    s  | du s	t | tsdS t | tr#| d j tj fdd| D dd}tjjj|dd}tj||dd\}}tjj	||}|du rStj
| dd}	tj
|dd}
ng|j\}}|jd ||  }|dddddddf |||||fd|| }tj| | ddtj|dd }	|ddddddf ||||fd| }tj|| ddtj|dd }
t|	|
d }|| S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   c                    s   g | ]}|  qS r0   )r6   )r   
layer_gatecompute_devicer0   r1   r   >  rm  z,load_balancing_loss_func.<locals>.<listcomp>rX   r3   )r=  r?   r  r'   r   r   r   r   r  r  r9   r   r@   r   r   r6   r  r   )r  r   r   r   concatenated_gate_logitsr  rO   r  r  tokens_per_expertrouter_prob_per_expertr   r  rq   expert_attention_mask router_per_expert_attention_maskoverall_lossr0   r  r1   load_balancing_loss_func  s>   



r  c                       s  e Zd ZdgZddiZddgdgfiZ fddZdd	 Zd
d Zdd Z	dd Z
dd Zdd Zee												d&deej deej deej deeej  deej deej dee dee dee dee d eej d!eeejf d"ee d#efd$d%Z  ZS )'MiniMaxForCausalLMzlm_head.weightlm_headcolwise_repr;   logitsc                    sX   t  | t|| _|j| _tj|j|jdd| _|j	| _	|j
| _|j| _|   d S r   )r$   r%   rl  r9  ro  r   rv   r,   r  router_aux_loss_coefr   r   r   rt  r   r.   r0   r1   r%   t  s   
zMiniMaxForCausalLM.__init__c                 C      | j jS rH   r9  rp  rA   r0   r0   r1   rw    rc   z'MiniMaxForCausalLM.get_input_embeddingsc                 C      || j _d S rH   r  rz  r0   r0   r1   r{       z'MiniMaxForCausalLM.set_input_embeddingsc                 C   ru  rH   r  rA   r0   r0   r1   get_output_embeddings  rx  z(MiniMaxForCausalLM.get_output_embeddingsc                 C   ry  rH   r  )r+   new_embeddingsr0   r0   r1   set_output_embeddings  r|  z(MiniMaxForCausalLM.set_output_embeddingsc                 C   ry  rH   r9  )r+   decoderr0   r0   r1   set_decoder  r|  zMiniMaxForCausalLM.set_decoderc                 C   ru  rH   r  rA   r0   r0   r1   get_decoder  rx  zMiniMaxForCausalLM.get_decoderNr   r}  r   r   r:  r~  labelsr3  r1  r  r2  r   logits_to_keepr   r   c                 K   s  |dur|n| j j}|
dur|
n| j j}
|	dur|	n| j j}	| jd||||||||	|
|d
|}|j}t|tr?t| dn|}| 	|dd|ddf }d}|dura| j
||| jfi |}d}|
r~t|j| j| j|}|dur~|| j||j 7 }t||||j|j|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MiniMaxForCausalLM

        >>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)
r}  r   r   r:  r~  r3  r1  r  r2  r   )lossaux_lossr  r:  r;   r  r  r0   )rg   r1  r2  r  r9  r  r=  rd   slicer  loss_functionro  r  r  r   r   r  r6   r  r   r:  r;   r  )r+   r}  r   r   r:  r~  r  r3  r1  r  r2  r   r  r   r6  r;   slice_indicesr  r  r  r0   r0   r1   r>     sX   )zMiniMaxForCausalLM.forward)NNNNNNNNNNNr   )rC   rD   rE   _tied_weights_keys_tp_plan_pp_planr%   rw  r{  r  r  r  r  r   r   r   r'   r   re   r  r7  r   r   rd   r   r  r   r>   rF   r0   r0   r.   r1   r  n  sl    	
r  a  
    The MiniMax Model transformer with a sequence classification head on top (linear layer).

    [`MiniMaxForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    )custom_introc                          e Zd Z fddZdd Zdd Zee									ddee	j
 d	ee	j d
ee	j
 dee dee	j dee	j
 dee dee dee defddZ  ZS ) MiniMaxForSequenceClassificationc                    s@   t  | |j| _t|| _tj|j| jdd| _| 	  d S r   )
r$   r%   
num_labelsrl  r9  r   rv   r,   scorert  r   r.   r0   r1   r%     s
   
z)MiniMaxForSequenceClassification.__init__c                 C   r  rH   r  rA   r0   r0   r1   rw  
  rc   z5MiniMaxForSequenceClassification.get_input_embeddingsc                 C   r  rH   r  rz  r0   r0   r1   r{    r  z5MiniMaxForSequenceClassification.set_input_embeddingsNr}  r   r   r:  r~  r  r3  r1  r  r   c
              
   C   s(  | j ||||||||	d}
|
j}| |}|dur|jd }n|jd }| jjdu r2|dkr2td| jjdu r;d}n1|dur`|| jjk|jt	j
}t	j|jd |jt	j
d}|| d}nd}t| jj d |t	j||jd	|f }d}|dur| j|||| jd
}t|||
j|
j|
jdS )  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r   r   r:  r~  r3  r1  r  Nr   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r3   )r  r5   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r  )r  r  pooled_logitsrg   )r  r  r:  r;   r  )r9  r  r  r@   rg   rn  r  r6   r  r'   int32r~   argmaxr  r  r/   rC   r  r   r:  r;   r  )r+   r}  r   r   r:  r~  r  r3  r1  r  transformer_outputsr;   r  r   last_non_pad_tokennon_pad_masktoken_indicesr  r  r0   r0   r1   r>     sL   


z(MiniMaxForSequenceClassification.forward	NNNNNNNNN)rC   rD   rE   r%   rw  r{  r   r   r   r'   r   re   r   r7  r   r   r>   rF   r0   r0   r.   r1   r    sH    		
r  c                       r  )MiniMaxForTokenClassificationc                    s|   t  | |j| _t|| _t|dd d ur|j}nt|dd d ur'|j}nd}t	|| _
t|j|j| _|   d S )Nclassifier_dropouthidden_dropoutg?)r$   r%   r  rl  r9  ro   r  r  r   Dropoutr   rv   r,   r  rt  )r+   rg   r  r.   r0   r1   r%   X  s   
z&MiniMaxForTokenClassification.__init__c                 C   r  rH   r  rA   r0   r0   r1   rw  h  rc   z2MiniMaxForTokenClassification.get_input_embeddingsc                 C   r  rH   r  rz  r0   r0   r1   r{  k  r  z2MiniMaxForTokenClassification.set_input_embeddingsNr}  r   r   r:  r~  r  r3  r1  r  r   c
              
   C   sd   | j ||||||||	d}
|
j}| |}| |}d}|dur(| ||| j}t|||
j|
jdS )r  r  N)r  r  r;   r  )	r9  r  r   r  r  rg   r   r;   r  )r+   r}  r   r   r:  r~  r  r3  r1  r  r6  sequence_outputr  r  r0   r0   r1   r>   n  s,   


z%MiniMaxForTokenClassification.forwardr  )rC   rD   rE   r%   rw  r{  r   r   r   r'   r   re   r   r7  r   r   r>   rF   r0   r0   r.   r1   r  V  sH    	
r  c                       s   e Zd ZdZ fddZdd Zdd Zee									dd	e	e
j d
e	e
j de	e
j de	eeee
j f  de	e
j de	e
j de	e
j de	e de	e defddZ  ZS )MiniMaxForQuestionAnsweringr9  c                    s2   t  | t|jd| _t|| _|   d S )Nr2   )	r$   r%   r   rv   r,   
qa_outputsrl  r9  rt  r   r.   r0   r1   r%     s   
z$MiniMaxForQuestionAnswering.__init__c                 C   r  rH   r  rA   r0   r0   r1   rw    rc   z0MiniMaxForQuestionAnswering.get_input_embeddingsc                 C   r  rH   r  rz  r0   r0   r1   r{    r  z0MiniMaxForQuestionAnswering.set_input_embeddingsNr}  r   r   r:  r~  start_positionsend_positionsr1  r  r   c
              	   K   s   | j |||||||	d}|j}| |}|jddd\}}|d }|d }d }|d urA|d urA| j||||fi |
}t||||j|j	dS )N)r   r   r:  r~  r1  r  r   r3   rX   )r  start_logits
end_logitsr;   r  )
r9  r  r  r   r  r   r  r   r;   r  )r+   r}  r   r   r:  r~  r  r  r1  r  r   r6  r  r  r  r  r  r0   r0   r1   r>     s0   

z#MiniMaxForQuestionAnswering.forwardr  )rC   rD   rE   rF  r%   rw  r{  r   r   r   r'   r   re   r   r   r  r7  r   r   r>   rF   r0   r0   r.   r1   r    sJ    	
r  )r8  rl  r  r  r  r  rJ   )r   )Nr2   N)Ltypingr   r   r   r'   torch.nn.functionalr   r   r   activationsr   cache_utilsr   r	   
generationr
   integrationsr   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   r   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   configuration_minimaxr    
get_loggerrC   r  Moduler"   rG   rf   r   r   re   rd   r   r   r   r   r   r   r  r8  rR  rl  r  r?   r  r  r  r  r  __all__r0   r0   r0   r1   <module>   s    
0 

>Ca" 
R VF>