o
    wiE                    @   sX  d dl mZmZmZ d dlZd dlm  mZ d dlmZ d dl	m
  m  mZ d dlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* e( rd dl+m,Z, d dl-m.Z.m/Z/ ndZ,e' rd dl0m1Z1m2Z2 nd\Z2Z1e$ rd dl3m4Z4 ddl5m6Z6 e%7e8Z9dd Z:dVddZ;dej<de=dej<fddZ>	 dWd!ej?d"ej<d#ej<d$ej<d%eej< d&e@d'e@fd(d)ZAG d*d+ d+ej?ZBG d,d- d-ejCZCd.ej<d/e=fd0d1ZDd2d3 ZEd4d5 ZFeGe,e1e2fZHd6d7 ZIG d8d9 d9ej?ZJG d:d; d;ejj?ZKG d<d= d=ej?ZLG d>d? d?ej?ZMG d@dA dAej?ZNG dBdC dCej?ZOG dDdE dEej?ZPG dFdG dGeZQe"G dHdI dIe ZRG dJdK dKej?ZSe"G dLdM dMeRZT		N	dXdOeej<eUej< df dPee= d%eej< deej<e=f fdQdRZVG dSdT dTeReZWg dUZXdS )Y    )CallableOptionalUnionN)nn)ACT2FN   )Cache)GenerationMixin)AttentionMaskConverter)GradientCheckpointingLayer)BaseModelOutputWithPastMoeCausalLMOutputWithPastMoeModelOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)auto_docstringcan_return_tupleis_torch_flex_attn_availablelogging)is_causal_conv1d_availableis_mamba_2_ssm_available   )GraniteMoeHybridConfig)selective_state_update)mamba_chunk_scan_combined mamba_split_conv1d_scan_combined)causal_conv1d_fncausal_conv1d_updateNN)	BlockMask)make_flex_block_causal_maskc                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..N   dim)shapetorchcat)xx1x2 r-   {/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/granitemoehybrid/modeling_granitemoehybrid.pyrotate_half@   s   r/   c                 C   sD   | |}| |}| | t| |  }|| t||  }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer/   )qkcossinposition_idsunsqueeze_dimq_embedk_embedr-   r-   r.   apply_rotary_pos_embG   s
   

r9   hidden_statesn_repreturnc                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r'   expandreshape)r:   r;   batchnum_key_value_headsslenhead_dimr-   r-   r.   	repeat_kvb   s
   0rC           modulequerykeyvalueattention_maskscalingdropoutc                 K   s   t || j}t || j}	t||dd| }
|d ur3|d d d d d d d |jd f }|
| }
tjj|
dtj	d
|j}
tjj|
|| jd}
t|
|	}|dd }||
fS )Nr$   r   r#   )r&   dtype)ptrainingr   )rC   num_key_value_groupsr(   matmul	transposer'   r   
functionalsoftmaxfloat32torM   rK   rO   
contiguous)rE   rF   rG   rH   rI   rJ   rK   kwargs
key_statesvalue_statesattn_weightscausal_maskattn_outputr-   r-   r.   eager_attention_forwardn   s   
&r^   c                       s   e Zd ZdZdedef fddZ						ddejd	e	ej d
e	ej
 de	e dede	ej
 de	eejejf  deeje	ej e	eej  f fddZ  ZS )GraniteMoeHybridAttentionz=Multi-headed attention from 'Attention Is All You Need' paperconfig	layer_idxc                    s   t    || _|| _|d u rtd| jj d |j| _|j	| _	|j
| _| j	| j | _|j| _| j| j | _d| _|j| _| j| j | j	krUtd| j	 d| j dtj| j	| j| j |jd| _tj| j	| j| j |jd| _tj| j	| j| j |jd| _tj| j	| j	|jd| _d S )NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).bias)super__init__r`   ra   loggerwarning_once	__class____name__attention_dropouthidden_sizenum_attention_heads	num_headsrB   r@   rP   	is_causalattention_multiplierrJ   
ValueErrorr   Linearattention_biasq_projk_projv_projo_projselfr`   ra   rh   r-   r.   re      s2   

z"GraniteMoeHybridAttention.__init__NFr:   rI   r5   past_key_value	use_cachecache_positionposition_embeddingsr<   c                 K   sF  |  \}	}
}| |}| |}| |}||	|
| j| jdd}||	|
| j| jdd}||	|
| j| jdd}|d urF|nd\}}|d urWt	||||\}}|d url|||d}|
||| j|\}}t}| jjdkrzt| jj }|| ||||f| jsdn| j| jd|\}}||	|
d}| |}|||fS )	Nr   r$   r    )r4   r3   r|   eagerrD   )rK   rJ   r#   )sizers   rt   ru   viewrm   rB   rR   r@   r9   updatera   r^   r`   _attn_implementationr   rO   rj   rJ   rv   )rx   r:   rI   r5   rz   r{   r|   r}   rX   bszq_len_query_statesrY   rZ   r3   r4   cache_kwargsattention_interfacer]   r[   r-   r-   r.   forward   s>   





z!GraniteMoeHybridAttention.forward)NNNFNN)ri   
__module____qualname____doc__r   intre   r(   Tensorr   
LongTensorr   booltupler   __classcell__r-   r-   ry   r.   r_      s4    #
r_   c                       s.   e Zd ZdZejdfdef fddZ  ZS ) HybridMambaAttentionDynamicCachea  
    A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache
    (which has a constant shape regardless of seq_len).

    This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states`
    and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor
    For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`,
    while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors).
    For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors),
    while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`,
    and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`.
    Nr`   c                    sB  t  | | |j| _d| _|j}|j}g | _g | _g | _t	|j
D ]^}| j| dkr\|  jtj |j|j d|j |  ||dg7  _|  jtj |j|j||dg7  _q$|  jtjg g  dg7  _|  jtjg g  dg7  _| j| q$ fddt	|j
D | _ fddt	|j
D | _d S )	NFmambar$   devicerM   r   c                        g | ]}t jg g  d qS r   r(   tensor.0r   
batch_sizer   r-   r.   
<listcomp>       z=HybridMambaAttentionDynamicCache.__init__.<locals>.<listcomp>c                    r   r   r   r   r   r-   r.   r     r   )rd   re   layers_block_typehas_previous_statemamba_d_convmamba_d_stateconv_states
ssm_statestransformer_layersrangenum_hidden_layersr(   zerosmamba_expandrk   mamba_n_groupsmamba_n_headsmamba_d_headr   append	key_cachevalue_cache)rx   r`   r   rM   r   conv_kernel_sizessm_state_sizeiry   r   r.   re      sD   	
   z)HybridMambaAttentionDynamicCache.__init__)	ri   r   r   r   r(   float16r   re   r   r-   r-   ry   r.   r      s    "r   input_tensorpad_sizec                 C   sH   t | jdkrddddd|ddfnddd|ddf}tjjj| |dddS )z
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
       r   constant)moderH   )lenr'   r(   r   rS   pad)r   r   	pad_shaper-   r-   r.   pad_tensor_by_size  s   2r   c                 C   sX   t | |} t| jdkr| | jd d|| jd S | | jd d|| jd | jd S )z
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   r#   r$   )r   r   r'   r>   )r   r   
chunk_sizer-   r-   r.   reshape_into_chunks&  s   
r   c                 C   s   |  d}| d jg |   |R  } tjtj||| jtjddd}| | d} tj| dd}tjtj||| jtjddd}|| tj	 }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r#   .Nr   diagonalr   rL   r%   )
r   r=   r(   trilonesr   r   masked_fillcumsuminf)r   r   masktensor_segsumr-   r-   r.   segment_sum:  s   
  r   c                 C   sN   |dur%|j d dkr%|j d dkr%| j}| |dddddf  |} | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    Nr   r   )r'   rM   rV   )r:   rI   rM   r-   r-   r.   apply_mask_to_padding_statesQ  s   $ r   c                       s   e Zd ZdZdedef fddZ				ddejde	e
 d	e	ej d
e	ej de	ej f
ddZ			dde	e
 d	e	ej d
e	ej fddZ				dde	e
 d	e	ej d
e	ej de	ej fddZ  ZS )GraniteMoeHybridMambaLayeruO  
    Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)

    The are a few differences between this and Mamba2Mixer:
    - The variable use_precomputed_states is slightly different due to the HybridCache structure
    - There's a few non-obvious bugs fixed with batching in the slow path that exist in main
    - Some extra variables that our layer doesn't need have been removed
    - We ported most of the refactors in https://github.com/huggingface/transformers/pull/35154, which is (as of Dec 18, 2024) unmerged
    r`   ra   c                    s  t    |j| _|j| _|j| _|j| _t	|j
| j | _|| _|j| _|j| _t|j | _|j| _|j| _|j| _|j| _|j| _dtdf| _d| _d| _ | jd| j | j  | _!t"j#| j!| j!|j| j| j!| jd d| _$| j| j! | j }t"j%| j|| jd| _&t"'t()| j| _*t(+d| jd }t"'t(,|| _-d	| j-_.t/| j| jd
| _0t"'t()| j| _1d	| j1_.t"j%| j| j| jd| _2t3st45d d S t45d d S )NrD   r   gMbP?g?r$   r   )in_channelsout_channelsrc   kernel_sizegroupspaddingrb   Tepsa  The fast path is not available because on of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1dzOThe fast path for GraniteMoeHybrid will be used when running the model on a GPU)6rd   re   r   rm   rk   r   r   r   r   r   r   intermediate_sizera   mamba_conv_biasuse_conv_bias
hidden_act
activationr   actmamba_proj_biasuse_biasrms_norm_epslayer_norm_epsilonr   n_groupsr   rB   mamba_chunk_sizer   floattime_step_limittime_step_mintime_step_maxconv_dimr   Conv1dconv1drq   in_proj	Parameterr(   r   dt_biasarangelogA_log_no_weight_decayGraniteMoeHybridRMSNormGatednormDout_projis_fast_path_availablerf   rg   )rx   r`   ra   projection_sizeAry   r-   r.   re   k  s\   

	z#GraniteMoeHybridMambaLayer.__init__Nr:   cache_paramsr|   rI   seq_idxc                 C   s  t ||}| |}|j\}}}	| j| j }
|d uoD|joD|dkoD|j| j jd |j| j jd   ko8|kn  oD|d uoD|d dk}|r)|	dj
| j| j| jgdd\}}}t||j| j | jj	d| jj| j}tj
|| j|
|
gdd\}}}t| j  }|d d d df d d d d d f d| j| jjtjd}|d d d d d f dd| j}| jd d d df d| j}| jd d d df d| j}||| j|jd | j }||| j|jd | j }||| j| j}t|j| j ||||||d |dd
}||| j| j }| ||}|  |d d d df }|S t| j  }| j!d	td
fkr>i nd| j!i}| j"r||d u r|t#|| jj	d| jj| j|f| j| j$|| j| jj| jj%| j j| j j| j| jddd|}|S |j
| j| j| jgdd\}}}|d ur|&dd}t'j()|| j*|jd  df}|j| j +| | jdvr| ,| |&dddd |f &dd}nt-|&dd| jj	d| jj| j|d&dd}t ||}tj
|| j|
|
gdd\}}}t.|||d| j|||||| jd|||| jdf| j$| jd |d| jdd|\}}|d ur:|d ur:|j| j +| |||d}| ||}|  |}|S )Nr   r   r#   r%   .rM   T)zr   dt_softplusrD   r   dt_limitF)r   r   r   r   rmsnorm_weightrmsnorm_epsoutproj_weightoutproj_biasheaddimngroupsnorm_before_gatereturn_final_statesr$   )siluswish)r*   weightrc   r   r   )r   r   r   r   r  r   r   )/r   r   r'   r   r   r   r   ra   r   squeezesplitr   r   rm   r   r   r  rc   r   r(   expr   r   r=   rB   rV   rU   r   r   r   r   r   r   r   rO   r   r   variance_epsilonrR   r   rS   r   r   copy_r   r   r   )rx   r:   r   r|   rI   r   projected_statesr   seq_lenr   groups_time_state_sizeuse_precomputed_statesgatehidden_states_B_CdtBCr   r   r   hidden_states_reshapedoutdt_limit_kwargshidden_states_B_C_transposedr   scan_output	ssm_stater-   r-   r.   cuda_kernels_forward  s  
	




<"
^"V
$




z/GraniteMoeHybridMambaLayer.cuda_kernels_forwardc           3   
      s  |j \}}}|j}t||}|}	|	jjjjgdd\}
}}|d uoQ|joQ|dkoQ|j	j
 j d |jj
 j d   koE|kn  oQ|d uoQ|d dk}|r|j	j
 jddd|j	j
< |d d dd d f |j	j
 j|j	j
 d d d d df< |j	j
 jjjjd}tj|jjd dd}jr|jj }|}n8|d ur|dd}tj|j|j d  df}|j	j
 | |dddd |f dd}t||}tj|jjj jj gdd\}}}tj !  }|r[|jj
 j}|d d dd d f d d d df }|dd"||j d j#}j$d	 "j$j d j#}tjj%|||j }t&|j'd j'd }|d
 "jj#jjtj(d}t|d	 | j|d}|)|jddd d d f }|"|jjj |j d * }|)|d|j d }|d	 |dd d d f  }|)|dj#}||d	  j|d}|jj
 |jj
 | |  |)|jddd d d f }|"|jjj |j d * }|)|d|j d }|jj
 j|j|jd}|+|j j#j}|+|j jd}t,||}|+|jj#}j-d	 "j-j d j#}|||  |j}|)|dd d d df }ntj%|j$ }t&|j'd j'd }|)||dj#! }|)||dj! }|)||dj! }|j.jj djd}|j.jj djd}j/|j/  j/  j-d	 t0|  }||d	  }||j| } fdd||||fD \}}}}|1dddd}tj2|dd}tt3|} |d d d d d d d d d d d f |d d d d d d d d d d d f  }!|!jdd}"|"d	 | 1dddddd	  }#|#jdd}$|$d	 |d d d d d f  jdd}%t|d d d d d d dd f | }&||&1ddddd	  }'|'dd d d f |d	  jdd}(|r|jj
 d d d df j|(jd})nt4|(d d d df })tj5|)|(gdd}(tt3tj|d d d d d d df d}*|*dd}*|*d
 |(d d d d d df  jdd}+|+d d d df |+d d df }(},t|}-|dd d d f |(d d d d d df  }.|-1dddd}/|.d|/d	  }0|%|0 }|)|djj#}|| } dkrB|d d d |d d d d f }|)||d}|,d ur\|d ur\|jj
 |, 6||
}17|1|}2|2S )Nr#   r%   r   r   )shiftsdimsr   r$   .r   ).NNr   r   )r&   output_sizec                    s   g | ]	}t | jqS r-   )r   r   )r   tr   rx   r-   r.   r     s    z<GraniteMoeHybridMambaLayer.torch_forward.<locals>.<listcomp>r   r   rL   )r   r   )8r'   rM   r   r   r	  r   r   rm   r   r   ra   r   rollrV   r   r   r  r(   sumr  r   rc   r   rR   r   rS   r   r   r  r   r   r
  r   r   r=   rB   r   softplusclampr   rU   r>   rW   r   bmmr   repeat_interleaver   r   permuter   r   
zeros_liker)   r   r   )3rx   input_statesr   r|   rI   r   r  r   rM   r  r  r  r  r  r   r  r:   r  r  r   cache_devicer   dAdBdBxr   ssm_states_reshaped
C_reshapedyr   
D_residualA_cumsumLG_intermediateGM_intermediateMY_diagdecay_statesB_decaystatesprevious_statesdecay_chunk
new_statesr  state_decay_outC_times_statesstate_decay_out_permutedY_offr  contextualized_statesr-   r!  r.   torch_forwardX  s   


@,
$"$$$P&*"&0(&
*
 z(GraniteMoeHybridMambaLayer.torch_forwardc                 K   s   t rd| jjjjv r| |||||S |d urtd|j}|d ur@|jd dkr@|jd dkr@||d d d d d f  	|}| 
||||S )Ncudaz\`seq_idx` support requires fast path support. Please install `mamba_ssm` and `causal_conv1d`r   r   )r   r   r  r   typer  NotImplementedErrorrM   r'   rV   rE  )rx   r:   r   r|   rI   r   rX   rM   r-   r-   r.   r   '  s   	$ z"GraniteMoeHybridMambaLayer.forward)NNNN)NNN)ri   r   r   r   r   r   re   r(   r   r   r   r   	IntTensorr  rE  r   r   r-   r-   ry   r.   r   ]  sV    F
 .
 Sr   c                       s(   e Zd Zd fdd	ZdddZ  ZS )	r   ư>c                    s&   t    tt|| _|| _d S Nrd   re   r   r   r(   r   r  r  rx   rk   r   ry   r-   r.   re   ?  s   

z%GraniteMoeHybridRMSNormGated.__init__Nc                 C   sj   |j }|tj}|d ur|tj|tj }|djddd}|t	|| j
  }| j|| S Nr$   r#   T)keepdim)rM   rV   r(   rU   r   rS   r  powmeanrsqrtr  r  )rx   r:   r  input_dtypevariancer-   r-   r.   r   D  s   z$GraniteMoeHybridRMSNormGated.forwardrJ  rK  )ri   r   r   re   r   r   r-   r-   ry   r.   r   >  s    r   c                       s<   e Zd ZdZdef fddZdejdejfddZ  Z	S )	GraniteMoeHybridMLPz~
    MLP layer for shared experts

    Args:
        config:
            Configuration object with model hyperparameters.
    r`   c                    sZ   t    |j| _|j| _t|j | _tj	| j| jd dd| _
tj	| j| jdd| _d S )Nr$   Frb   )rd   re   rk   
input_sizeshared_intermediate_sizer   r   r   r   rq   input_linearoutput_linearrx   r`   ry   r-   r.   re   Y  s   
zGraniteMoeHybridMLP.__init__r:   r<   c                 C   s<   |  |}|jddd}| |d |d  }| |}|S )Nr$   r#   r%   r   r   )rY  chunkr   rZ  )rx   r:   chunked_hidden_statesr-   r-   r.   r   b  s
   

zGraniteMoeHybridMLP.forward)
ri   r   r   r   r   re   r(   r   r   r   r-   r-   ry   r.   rV  P  s    	rV  c                       s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	GraniteMoeHybridRMSNormrJ  c                    s&   t    tt|| _|| _dS )zF
        GraniteMoeHybridRMSNorm is equivalent to T5LayerNorm
        NrL  rM  ry   r-   r.   re   k  s   

z GraniteMoeHybridRMSNorm.__init__c                 C   sJ   |j }|tj}|djddd}|t|| j  }| j|| S rN  )	rM   rV   r(   rU   rP  rQ  rR  r  r  )rx   r:   rS  rT  r-   r-   r.   r   s  s
   zGraniteMoeHybridRMSNorm.forwardc                 C   s   t | jj d| j S )Nz, eps=)r   r  r'   r  rx   r-   r-   r.   
extra_reprz  s   z"GraniteMoeHybridRMSNorm.extra_reprrU  )ri   r   r   re   r   r`  r   r-   r-   ry   r.   r^  j  s    r^  c                       s6   e Zd Zdedededdf fddZdd	 Z  ZS )
GraniteMoeHybridParallelExpertsnum_expertsrW  r  r<   Nc                    s6   t    tt|||| _|| _|| _|| _	dS )a  
        Initialize the GraniteMoeHybridParallelExperts module.
        The experts weights are stored in [num_experts, output_size, input_size] format. Such that it's compatible with
        many MoE libraries, such as [Megablock](https://github.com/databricks/megablocks) and
        [ScatterMoE](https://github.com/shawntan/scattermoe), as well as the
        [MoE kernel](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/fused_moe/fused_moe.py)
        used in vllm.

        Args:
            num_experts (int):
                Number of experts.
            input_size (int):
                Size of the input.
            output_size (int):
                Size of the output.
        N)
rd   re   r   r   r(   emptyr  rb  rW  r  )rx   rb  rW  r  ry   r-   r.   re     s
   

z(GraniteMoeHybridParallelExperts.__init__c                 C   sP   |j |dd}g }t| jD ]}|t|| | j|  qtj|dd}|S )a  
        Forward pass of the GraniteMoeHybridParallelExperts module.

        Args:
            inputs (Tensor):
                Input tensor.
            expert_size:
                Expert size information.

        Returns:
            Tensor: Output tensor.
        r   r%   )	r	  r   rb  r   Flinearr  r(   r)   )rx   inputsexpert_size
input_listoutput_listr   resultsr-   r-   r.   r     s   z'GraniteMoeHybridParallelExperts.forwardri   r   r   r   re   r   r   r-   r-   ry   r.   ra  ~  s    ra  c                       s2   e Zd Zdededef fddZdd Z  ZS )GraniteMoeHybridTopKGatingrW  rb  top_kc                    s2   t    || _|| _|| _tj||dd| _dS )a  
        Initialize the top-k gating mechanism.
        Args:
            input_size (`int`):
                Size of the input.
            num_experts (`int`):
                Number of experts.
            top_k (`int`):
                Number of top experts to select.
        Frb   N)rd   re   rb  rW  rm  r   rq   layer)rx   rW  rb  rm  ry   r-   r.   re     s
   
z#GraniteMoeHybridTopKGating.__init__c                 C   s   |  | }|j| jdd\}}tj|dd|}tj|d| j	g|j
|jd}|d|d}| d}| }| }	|	d\}
}|j| jdd}| }|| }|||||fS )Nr   r%   r   rM   r   trunc)rounding_mode)rn  r   topkrm  r(   rT   type_asr   r   rb  rM   r   scatterlongr#  tolistflattensortdiv)rx   r:   logitstop_k_logitstop_k_indicestop_k_gatesr   gatesrg  top_k_expertsr   index_sorted_expertsbatch_indexbatch_gatesr-   r-   r.   r     s   z"GraniteMoeHybridTopKGating.forwardrk  r-   r-   ry   r.   rl    s    rl  c                       s.   e Zd ZdZdef fddZdd Z  ZS )GraniteMoeHybridMoEz
    A Sparsely gated mixture of experts layer with 1-layer Feed-Forward networks as experts.

    Args:
        config:
            Configuration object with model hyperparameters.
    r`   c                    sl   t    |j| _|j| _t|j | _t|j	| j| jd | _
t|j	| j| j| _t| j|j	|jd| _d S )Nr$   )rW  rb  rm  )rd   re   rk   rW  r   r   r   r   ra  num_local_expertsrY  rZ  rl  num_experts_per_tokrouterr[  ry   r-   r.   re     s   
zGraniteMoeHybridMoE.__init__c                 C   s   |  \}}}|d|}| |\}}}}}	|| }
| |
|}|jddd}| |d |d  }| ||}||dddf  }tj|| | j	f|j
|jd}|d||}|||| j	}||	fS )a  
        Forward pass of the mixture of experts layer.

        Args:
            layer_input (Tensor):
                Input tensor.

        Returns:
            Tensor:
                Output tensor.
            Tensor:
                Router logits.
        r#   r$   r%   r   r   Nro  )r   r>   r  rY  r\  r   rZ  r(   r   rW  rM   r   	index_addr   )rx   layer_inputr   lengthemb_sizer   r  r  rg  router_logitsexpert_inputsr:   r]  expert_outputsr   layer_outputr-   r-   r.   r     s   zGraniteMoeHybridMoE.forward)ri   r   r   r   r   re   r   r   r-   r-   ry   r.   r    s    r  c                       s   e Zd Zdedef fddZ							ddejdeej d	ee	 d
ee
 dee
 deej dee
 deeejejf  deejeeejejf  f fddZ  ZS )GraniteMoeHybridDecoderLayerr`   ra   c                    s   t    |j| _d | _|jdkrt|| _t|j|jd| _	t|j|jd| _
|j| _t|| _d | _|j| dkrBt||| _nt||| _|j| | _t|dddk| _d S )Nr   r   r   r  )rd   re   rk   	self_attnr  r  block_sparse_moer^  r   input_layernormpost_attention_layernormresidual_multiplierrV  
shared_mlpr   r   r   r_   
layer_typegetattrhas_expertsrw   ry   r-   r.   re     s   



z%GraniteMoeHybridDecoderLayer.__init__NFr:   rI   rz   output_attentionsr{   r|   output_router_logitsr}   r<   c	              
   K   s   |}
|  |}| jdur| j||||d}d}n| jd|||||||d|	\}}}|
|| j  }|}
| |}| jrK| |\}}|| | }n| |}d}|
|| j  }|f}|rc||f7 }|rj||f7 }|rq||f7 }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
            output_router_logits (`bool`, *optional*):
                Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
                should not be returned during inference.
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        N)r:   r|   r   rI   )r:   rI   rz   r  r{   r|   r}   r-   )r  r   r  r  r  r  r  r  )rx   r:   rI   rz   r  r{   r|   r  r}   rX   residualself_attn_weightsr   moe_hidden_statesr  outputsr-   r-   r.   r   /  sL   %






z$GraniteMoeHybridDecoderLayer.forward)NNFFNFN)ri   r   r   r   r   re   r(   r   r   r   r   r   r   FloatTensorr   r   r-   r-   ry   r.   r    s8    	r  c                   @   sD   e Zd ZeZdZdZdgZdgZdZ	dZ
dZdZdZdZdd ZdS )	GraniteMoeHybridPreTrainedModelmodelTr  past_key_valuesFc                 C   sZ  t |tjr|jjjd| jjd |jd ur|jj	  n=t |tj
r=|jjjd| jjd |jd ur<|jj|j 	  nt |trJ|jjd nt |trZ|jjjd| jjd t |tjrz|jjjd| jjd |jd urx|jj	  d S d S t |tr|jjd ttd|jd |j_|jjd d S t |tr|jjd d S d S )NrD   )rQ  stdg      ?r   )
isinstancer   rq   r  datanormal_r`   initializer_rangerc   zero_	Embeddingpadding_idxr^  fill_ra  r   r   r   r(   r   r   rm   r   r   r   )rx   rE   r-   r-   r.   _init_weights  s4   






z-GraniteMoeHybridPreTrainedModel._init_weightsN)ri   r   r   r   config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_2_supports_sdpa_supports_cache_class_supports_quantized_cache_supports_static_cache_is_statefulr  r-   r-   r-   r.   r    s    r  c                       s8   e Zd Zddef fddZe edd Z  Z	S )GraniteMoeHybridRotaryEmbeddingNr`   c                    s   t    t|dr|jd ur|jd|jd| _nd| _|j| _|j| _|| _	t
| j | _| | j	|\}| _| jd|dd | j| _d S )Nrope_scaling	rope_typerG  defaultinv_freqF)
persistent)rd   re   hasattrr  getr  max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr`   r   rope_init_fnattention_scalingregister_bufferr  original_inv_freq)rx   r`   r   r  ry   r-   r.   re     s   
z(GraniteMoeHybridRotaryEmbedding.__init__c           
      C   s   | j d d d d f  |jd dd|j}|d d d d d f  }t|jjtr6|jjdkr6|jjnd}t	j
|dd+ | |  dd}t	j||fdd	}| | j }| | j }	W d    n1 smw   Y  |j|jd
|	j|jd
fS )Nr   r#   r   mpscpuF)device_typeenabledr$   r%   r   )r  r   r=   r'   rV   r   r  rG  strr(   autocastrR   r)   r3   r  r4   rM   )
rx   r*   r5   inv_freq_expandedposition_ids_expandedr  freqsembr3   r4   r-   r-   r.   r     s   0&z'GraniteMoeHybridRotaryEmbedding.forwardrK  )
ri   r   r   r   re   r(   no_gradr   r   r   r-   r-   ry   r.   r    s
    r  c                       s8  e Zd Zdef fddZdd Zdd Zee											d$d	e	j
d
ee	j dee	j
 deeeee	j f  dee	j dee dee dee dee dee dee	j
 deeef fddZ	d%d
ee	jdf de	jde	jdedef
ddZed
e	jdedede	jde	jdefd d!Zd"d# Z  ZS )&GraniteMoeHybridModelr`   c                    s   t     j| _ j| _t j j| j| _t	 fddt
 jD | _t j jd| _d| _ j| _ j| _ j| _| j| j | _ j| _ j| _ j| _| jdkr]t nd | _|   d S )Nc                    s   g | ]}t  |qS r-   )r  )r   ra   r`   r-   r.   r         z2GraniteMoeHybridModel.__init__.<locals>.<listcomp>r   Frope)rd   re   pad_token_idr  
vocab_sizer   r  rk   embed_tokens
ModuleListr   r   layersr^  r   r   gradient_checkpointingembedding_multiplierrl   rm   rB   r  
rope_thetaposition_embedding_typer  
rotary_emb	post_initr[  ry   r  r.   re     s$   zGraniteMoeHybridModel.__init__c                 C      | j S rK  r  r_  r-   r-   r.   get_input_embeddings     z*GraniteMoeHybridModel.get_input_embeddingsc                 C   
   || _ d S rK  r  rx   rH   r-   r-   r.   set_input_embeddings     
z*GraniteMoeHybridModel.set_input_embeddingsN	input_idsrI   r5   r  inputs_embedsr{   r  output_hidden_statesr  return_dictr|   r<   c                 C   sT  |d ur|n| j j}|d ur|n| j j}|d ur|n| j j}|
d ur$|
n| j j}
|d u |d uA r4td| jrC| jrC|rCt	d d}|d u rL| 
|}|| j }|r\|d u r\t	d |d u rx|d urh| nd}tj|||jd  |jd}|d u r|d}| |||||}| ||}|}d }| jd ur| ||}|rdnd }|rdnd }|	rdnd }d }| jD ]L}|jd	kr|n|}|r||f7 }||||||||	|d
}|d }|r||rdnd }|r|d d ur||d f7 }|	r|d d ur||d f7 }q| |}|r||f7 }|r|jsd|_|r|nd }t|||||dS )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzGraniteMoeHybrid requires an initialized `HybridMambaAttentionDynamicCache` to return a cache. Because one was not provided, no cache will be returned.r   r   r   r-   r   )rI   rz   r  r{   r|   r  r}   r$   r#   T)last_hidden_stater  r:   
attentionsr  )r`   r  r  r{   use_return_dictrp   r  rO   rf   rg   r  r  get_seq_lengthr(   r   r'   r   r0   _update_causal_mask_update_mamba_maskr  r  r  r   r   r   )rx   r  rI   r5   r  r  r{   r  r  r  r  r|   past_seen_tokensr\   
mamba_maskr:   r}   all_hidden_statesall_self_attnsall_router_logitsnext_decoder_cachedecoder_layer
layer_masklayer_outputs
next_cacher-   r-   r.   r     s   








zGraniteMoeHybridModel.forwardFr!   r   c                 C   s:  | j jdkr|d ur|dk r|S d S | j jdkr&t|tjr$t|}|S |d ur.| nd}|d ur7|jnd}| j jdkrO|sO|sOt	j
|||| jdrOd S |j}|jd }	|r^| }
nt|tjri|jd	 n||	 d }
| j||	|
|||jd d
}| j jdkr|d ur|jjdv r|st|j}t	||}|S )Nflash_attention_2rD   flex_attentionr   Fsdpa)r  past_key_values_lengthis_trainingr   r#   )sequence_lengthtarget_lengthrM   r|   r   )rF  xpunpu)r`   r   anyr  r(   r   r"   r  is_compileabler
   _ignore_causal_mask_sdparO   rM   r'   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionr   rG  finfomin_unmask_unattended)rx   rI   r   r|   r  r  r  using_compilable_cacherM   r  r  r\   	min_dtyper-   r-   r.   r  o  sT   




z)GraniteMoeHybridModel._update_causal_maskr  r  rM   r   c                 K   sD  | dur|   dkr| }|S t|j}tj||f|||jd}|dkr+tj|dd}|tj||jd|ddk9 }|ddddddf 	|ddd}| dur|
 }| jd }	|ddddddd|	f | ddddddf |j }
|
dk}
|ddddddd|	f |
||ddddddd|	f< |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        Nr   )
fill_valuerM   r   r   r   r   r#   r   )r&   r(   r  r  fullr   triur   r>   r=   cloner'   rV   r   )rI   r  r  rM   r|   r   rX   r\   r  mask_lengthpadding_maskr-   r-   r.   r
    s,    $
6  zKGraniteMoeHybridModel._prepare_4d_causal_attention_mask_with_cache_positionc                 C   s.   |}|d dks|durt |dkrd}|S )zv
        No need for zeroing states when
            1. Cached forward
            2. Attending to all inputs
        r   Nr   )r(   all)rx   rI   r|   r  r-   r-   r.   r    s   "z(GraniteMoeHybridModel._update_mamba_mask)NNNNNNNNNNN)F)ri   r   r   r   re   r  r  r   r   r(   r   r   r   r   r   listr  r   r   r   r   r  staticmethodr   rM   r
  r  r   r-   r-   ry   r.   r    s    	


D6r  r$   gate_logitsrb  c                    s  | du s	t | tsdS t | tr#| d j tj fdd| D dd}tjjj|dd}tj||dd\}}tjj	||}|du rStj
| dd}	tj
|dd}
ng|j\}}|jd ||  }|dddddddf |||||fd|| }tj| | ddtj|dd }	|ddddddf ||||fd| }tj|| ddtj|dd }
t|	|
d }|| S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   c                    s   g | ]}|  qS r-   )rV   )r   
layer_gatecompute_devicer-   r.   r     r  z,load_balancing_loss_func.<locals>.<listcomp>r%   r#   )r  r   r   r(   r)   r   rS   rT   rr  one_hotrQ  r   r'   r=   r>   rV   r#  r0   )r  rb  rm  rI   concatenated_gate_logitsrouting_weightsr   selected_expertsexpert_masktokens_per_expertrouter_prob_per_expertr   r  r   expert_attention_mask router_per_expert_attention_maskoverall_lossr-   r  r.   load_balancing_loss_func  s>   



r'  c                        sF  e Zd ZdgZdef fddZdd Zdd Zd	d
 Zdd Z	dd Z
dd Ze													d*deej deej deej deeeeej f  deej deej dee dee dee dee dee deej deeejf d eeef fd!d"Zed#d$ Z						%d+d&d'Zd efd(d)Z  ZS ),GraniteMoeHybridForCausalLMzlm_head.weightr`   c                    sX   t  | t|| _|j| _tj|j|jdd| _|j	| _	|j
| _|j| _|   d S )NFrb   )rd   re   r  r  r  r   rq   rk   lm_headrouter_aux_loss_coefr  rb  r  r  r[  ry   r-   r.   re   K  s   
z$GraniteMoeHybridForCausalLM.__init__c                 C   s   | j jS rK  r  r  r_  r-   r-   r.   r  X  s   z0GraniteMoeHybridForCausalLM.get_input_embeddingsc                 C   s   || j _d S rK  r+  r  r-   r-   r.   r  [  s   z0GraniteMoeHybridForCausalLM.set_input_embeddingsc                 C   r  rK  r)  r_  r-   r-   r.   get_output_embeddings^  r  z1GraniteMoeHybridForCausalLM.get_output_embeddingsc                 C   r  rK  r,  )rx   new_embeddingsr-   r-   r.   set_output_embeddingsa  r  z1GraniteMoeHybridForCausalLM.set_output_embeddingsc                 C   r  rK  r  )rx   decoderr-   r-   r.   set_decoderd  r  z'GraniteMoeHybridForCausalLM.set_decoderc                 C   r  rK  r0  r_  r-   r-   r.   get_decoderg  r  z'GraniteMoeHybridForCausalLM.get_decoderNr   r  rI   r5   r  r  labelsr{   r  r  r  r  r|   logits_to_keepr<   c                 K   s  |dur|n| j j}|
dur|
n| j j}
|	dur|	n| j j}	|dur$|n| j j}| j||||||||	|
||d}|d }t|trGt| dn|}| 	|dd|ddf }|| j j
 }d}|duru| }| j||fd| j ji|}d}|
rt|r|jn|d | j| j|}|dur|| j||j 7 }|s|f|dd  }|
r|f| }|dur|f| S |S t||||j|j|j|jdS )ax  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeHybridForCausalLM

        >>> model = GraniteMoeHybridForCausalLM.from_pretrained("ibm/PowerMoE-3b")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm/PowerMoE-3b")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r  rI   r5   r  r  r{   r  r  r  r  r|   r   r  r#   r   )lossaux_lossrz  r  r:   r  r  )r`   r  r  r  r  r  r  r   slicer)  logits_scalingr   loss_functionr  r'  r  rb  r  r*  rV   r   r   r  r:   r  )rx   r  rI   r5   r  r  r4  r{   r  r  r  r  r|   r5  rX   r  r:   slice_indicesrz  r6  r7  outputr-   r-   r.   r   j  st   (
z#GraniteMoeHybridForCausalLM.forwardc                    s.   d}| D ]}|t  fdd|D f7 }q|S )Nr-   c                 3   s$    | ]}| d  |jV  qdS )r   N)index_selectrV   r   )r   
past_statebeam_idxr-   r.   	<genexpr>  s   " z=GraniteMoeHybridForCausalLM._reorder_cache.<locals>.<genexpr>)r   )r  r@  reordered_past
layer_pastr-   r?  r.   _reorder_cache  s   z*GraniteMoeHybridForCausalLM._reorder_cacheTc                 K   s  |d u }	|	s5|d us|d |j d kr"|d d |j d  d f }n!|j d |j d kr4|d d |f }nt| j|j d | j| jd}|d url|d u rl| dd }||dkd |	sl|d d |j d  d f }|d urw|	rwd|i}
nd| i}
|
	|||||d |
S )Nr#   r   r   r   r  r  )r5   r  r{   rI   r|   )
r'   r   r`   rM   r   ru  r   masked_fill_rW   r   )rx   r  r  rI   r  r|   r5   r{   rX   empty_past_kvmodel_inputsr-   r-   r.   prepare_inputs_for_generation  s8   
	z9GraniteMoeHybridForCausalLM.prepare_inputs_for_generationc                 C   s   dS )aG  
        Function overwritten as this class uses its own `HybridMambaAttentionDynamicCache`
        and do not need to initialize the Cache in advance in order to save memory
        (because no back and forth `to_legacy_cache` and `from_legacy_cache` will be performed
        for `HybridMambaAttentionDynamicCache`).
        Fr-   r_  r-   r-   r.   _supports_default_dynamic_cache  s   z;GraniteMoeHybridForCausalLM._supports_default_dynamic_cache)NNNNNNNNNNNNr   )NNNNNT)ri   r   r   _tied_weights_keysr   re   r  r  r-  r/  r2  r3  r   r   r(   r   r   r   r   r  r  r   r   r   r   r   r  rD  rH  rI  r   r-   r-   ry   r.   r(  H  s|    	

l

9r(  )r(  r  r  )Nr   )rD   )Nr$   N)Ytypingr   r   r   r(   torch.nn.functionalr   rS   rd  (transformers.models.jamba.modeling_jambamodelsjambamodeling_jambatransformers.activationsr   cache_utilsr   
generationr	   modeling_attn_mask_utilsr
   modeling_layersr   modeling_outputsr   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   utilsr   r   r   r   utils.import_utilsr   r   configuration_granitemoehybridr   +mamba_ssm.ops.triton.selective_state_updater   !mamba_ssm.ops.triton.ssd_combinedr   r   causal_conv1dr   r   !torch.nn.attention.flex_attentionr!   integrations.flex_attentionr"   
get_loggerri   rf   r/   r9   r   r   rC   Moduler   r^   r_   r   r   r   r   r  r   r   r   r   rV  r^  ra  rl  r  r  r  r  r  r   r'  r(  __all__r-   r-   r-   r.   <module>   s   


W9   d-0<r'"  &
R \