o
    i                    @   s  d dl mZmZmZmZmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z( ddl)m*Z* e( rd dl+m,Z, d dl-m.Z.m/Z/ ndZ,e' rd dl0m1Z1m2Z2 nd\Z2Z1e#3e4Z5G dd deddZ6G dd dZ7G dd dej8Z9d d! Z:d"ej;d#e<d$ej;fd%d&Z=	'dRd(ej8d)ej;d*ej;d+ej;d,eej; d-e>d.e>d/ee  fd0d1Z?dSd2d3Z@G d4d5 d5ej8ZAG d6d7 d7ejj8ZBd8ej;d9e<fd:d;ZCd<d= ZDd>d? ZEeFe,e1e2fZGd@dA ZHG dBdC dCej8ZIG dDdE dEej8ZJedFG dGdH dHej8ZKG dIdJ dJeZLe!G dKdL dLeZMe!G dMdN dNeMZNe!G dOdP dPeMeZOg dQZPdS )T    )AnyCallableOptional	TypedDictUnionN)nn)ACT2FN   )Cache)GenerationMixin)use_kernel_forward_from_hub)AttentionMaskConverter)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)deprecate_kwarg)is_causal_conv1d_availableis_mamba_2_ssm_available   )BambaConfig)selective_state_update)mamba_chunk_scan_combined mamba_split_conv1d_scan_combined)causal_conv1d_fncausal_conv1d_updateNNc                   @   s@   e Zd ZU dZejed< ejed< eed< eed< ejed< dS )BambaFlashAttentionKwargsa  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    Attributes:
        cu_seq_lens_q (`torch.LongTensor`)
            Gets cumulative sequence length for query state.
        cu_seq_lens_k (`torch.LongTensor`)
            Gets cumulative sequence length for key state.
        max_length_q (`int`):
            Maximum sequence length for query state.
        max_length_k (`int`):
            Maximum sequence length for key state.
        seq_idx (`torch.IntTensor):
            Index of each packed sequence.
    cu_seq_lens_qcu_seq_lens_kmax_length_qmax_length_kseq_idxN)	__name__
__module____qualname____doc__torch
LongTensor__annotations__int	IntTensor r4   r4   e/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/transformers/models/bamba/modeling_bamba.pyr%   @   s   
 

r%   F)totalc                   @   s   e Zd ZdZdZejdfdefddZ	ddej	dej	d	e
d
eeeef  deej	ej	f f
ddZdejfddZdd	ee
 de
fddZdS ) HybridMambaAttentionDynamicCachea  
    A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache
    (which has a constant shape regardless of seq_len).

    This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states`
    and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor
    For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`,
    while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors).
    For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors),
    while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`,
    and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`.
    FNconfigc                    s0  |j | _ d| _|j}|j}g | _g | _g | _t|jD ]^}| j | dkrS|  jt	j
 |j|j d|j |  ||dg7  _|  jt	j
 |j|j||dg7  _q|  jt	jg g  dg7  _|  jt	jg g  dg7  _| j| q fddt|jD | _ fddt|jD | _d S )	NFmamba   devicedtyper<   c                        g | ]}t jg g  d qS r>   r/   tensor.0_
batch_sizer<   r4   r5   
<listcomp>        z=HybridMambaAttentionDynamicCache.__init__.<locals>.<listcomp>c                    r?   r@   rA   rC   rF   r4   r5   rH      rI   )layers_block_typehas_previous_statemamba_d_convmamba_d_stateconv_states
ssm_statestransformer_layersrangenum_hidden_layersr/   zerosmamba_expandhidden_sizemamba_n_groupsmamba_n_headsmamba_d_headrB   append	key_cachevalue_cache)selfr8   rG   r=   r<   conv_kernel_sizessm_state_sizeir4   rF   r5   __init__i   sB   	
   z)HybridMambaAttentionDynamicCache.__init__
key_statesvalue_states	layer_idxcache_kwargsreturnc                 C   sz   | j | jd dkr|| j |< || j|< ntj| j | |gdd| j |< tj| j| |gdd| j|< | j | | j| fS )Nr   r:   dim)rZ   shaper[   r/   cat)r\   ra   rb   rc   rd   r4   r4   r5   update   s   
z'HybridMambaAttentionDynamicCache.updatebeam_idxc                 C   s   t t| jD ]V}| j| j}| j| d||| j|< | j| j}| j| d||| j|< | j| j}| j| d||| j|< | j| j}| j| d||| j|< qdS )zDReorders the cache for beam search, given the selected beam indices.r   N)	rQ   lenrZ   r<   index_selecttor[   rN   rO   )r\   rl   rc   r<   r4   r4   r5   reorder_cache   s    z.HybridMambaAttentionDynamicCache.reorder_cacher   c                 C   s:   || j vr
| j d n|}t| j|krdS | j| jd S )zYReturns the sequence length of the cached states. A layer index can be optionally passed.r   )rP   rm   rZ   ri   )r\   rc   r4   r4   r5   get_seq_length   s   z/HybridMambaAttentionDynamicCache.get_seq_lengthN)r   )r+   r,   r-   r.   is_compileabler/   float16r   r`   Tensorr2   r   dictstrr   tuplerk   r0   rp   rr   r4   r4   r4   r5   r7   Y   s$    +
r7   c                       sD   e Zd ZU ejed< ddef fddZe e	dd Z
  ZS )	BambaRotaryEmbeddinginv_freqNr8   c                    s   t    t|drt|jtr|jd|jd| _nd| _|j| _	|j| _
|| _t| j | _| | j|\}| _| jd|dd | j| _d S )Nrope_scaling	rope_typetypedefaultr{   F)
persistent)superr`   hasattr
isinstancer|   rw   getr}   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr8   r   rope_init_fnattention_scalingregister_bufferr{   original_inv_freq)r\   r8   r<   r{   	__class__r4   r5   r`      s   
zBambaRotaryEmbedding.__init__c           
      C   s   | j d d d d f  |jd dd|j}|d d d d d f  }t|jjtr6|jjdkr6|jjnd}t	j
|dd+ | |  dd}t	j||fdd	}| | j }| | j }	W d    n1 smw   Y  |j|jd
|	j|jd
fS )Nr   rf   r   mpscpuF)device_typeenabledr:   rg   r=   )r{   floatexpandri   ro   r<   r   r~   rx   r/   autocast	transposerj   cosr   sinr=   )
r\   xposition_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   r4   r4   r5   forward   s   0&zBambaRotaryEmbedding.forwardrs   )r+   r,   r-   r/   rv   r1   r   r`   no_gradr   r   __classcell__r4   r4   r   r5   rz      s   
 
rz   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..Nrf   r:   rg   )ri   r/   rj   )r   x1x2r4   r4   r5   rotate_half   s   r   hidden_statesn_repre   c                 C   s^   | j \}}}}|dkr| S | dddddddddf |||||} | ||| ||S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)ri   r   reshape)r   r   batchnum_key_value_headsslenhead_dimr4   r4   r5   	repeat_kv   s
   0r           modulequerykeyvalueattention_maskscalingdropoutkwargsc                 K   s   t || j}t || j}	t||dd| }
|d ur3|d d d d d d d |jd f }|
| }
tjj|
dtj	d
|j}
tjj|
|| jd}
t|
|	}|dd }||
fS )Nr:   r	   rq   rf   )rh   r=   )ptrainingr   )r   num_key_value_groupsr/   matmulr   ri   r   
functionalsoftmaxfloat32ro   r=   r   r   
contiguous)r   r   r   r   r   r   r   r   ra   rb   attn_weightscausal_maskattn_outputr4   r4   r5   eager_attention_forward   s   
&r   c                 C   s   | |}| |}|jd }| dd|f | d|df }}|dd|f |d|df }	}
|| t||  }|	| t|	|  }tj||gdd}tj||
gdd}||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    rf   .Nrg   )	unsqueezeri   r   r/   rj   )qkr   r   r   unsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embedr4   r4   r5   apply_rotary_pos_emb  s   


""r   c                       s   e Zd ZdZdedef fddZedddd		
	
ddej	de
ej	ej	f deej	 dee deej dee de
ej	ej	f fddZ  ZS )BambaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr8   rc   c                    s   t    || _|| _t|d|j|j | _|j|j | _	| jd | _
|j| _d| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j|j| j |jd| _tj|j| j |j|jd| _d S )Nr   g      Tbias)r   r`   r8   rc   getattrrU   num_attention_headsr   r   r   r   attention_dropout	is_causalr   Linearattention_biasq_projk_projv_projo_proj)r\   r8   rc   r   r4   r5   r`   3  s(   
zBambaAttention.__init__past_key_valuepast_key_values4.58new_nameversionNr   position_embeddingsr   cache_positionr   re   c                 K   s$  |j d d }g |d| jR }| ||dd}	| ||dd}
| ||dd}|\}}t|	|
||\}	}
|d urW|||d}||
|| j	|\}
}t
}| jjdkret| jj }|| |	|
||f| jsqdn| j| jd|\}}|jg |dR   }| |}||fS )Nrf   r   r:   )r   r   r   eagerr   )r   r   )ri   r   r   viewr   r   r   r   rk   rc   r   r8   _attn_implementationr   r   r   r   r   r   r   )r\   r   r   r   r   r   r   input_shapehidden_shapequery_statesra   rb   r   r   rd   attention_interfacer   r   r4   r4   r5   r   J  s8   


zBambaAttention.forwardr$   )r+   r,   r-   r.   r   r2   r`   r   r/   rv   ry   r   r
   r0   r   r   r   r   r4   r4   r   r5   r   0  s*    r   c                       s(   e Zd Zd fdd	ZdddZ  ZS )	BambaRMSNormGatedư>c                    s&   t    tt|| _|| _d S rs   r   r`   r   	Parameterr/   onesweightvariance_epsilonr\   rU   epsr   r4   r5   r`   x  s   

zBambaRMSNormGated.__init__Nc                 C   sj   |j }|tj}|d ur|tj|tj }|djddd}|t	|| j
  }| j|| S Nr:   rf   T)keepdim)r=   ro   r/   r   r   r   silupowmeanrsqrtr   r   )r\   r   gateinput_dtypevariancer4   r4   r5   r   }  s   zBambaRMSNormGated.forwardr   rs   r+   r,   r-   r`   r   r   r4   r4   r   r5   r   w  s    r   input_tensorpad_sizec                 C   sH   t | jdkrddddd|ddfnddd|ddf}tjjj| |dddS )z
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
       r   constant)moder   )rm   ri   r/   r   r   pad)r   r   	pad_shaper4   r4   r5   pad_tensor_by_size  s   2r  c                 C   sX   t | |} t| jdkr| | jd d|| jd S | | jd d|| jd | jd S )z
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r	   r   rf   r:   )r  rm   ri   r   )r   r   
chunk_sizer4   r4   r5   reshape_into_chunks  s   
r  c                 C   s   |  d}| d jg |   |R  } tjtj||| jtjddd}| | d} tj| dd}tjtj||| jtjddd}|| tj	 }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    rf   .Nr;   diagonalr   rq   rg   )
sizer   r/   trilr   r<   boolmasked_fillcumsuminf)r   r  masktensor_segsumr4   r4   r5   segment_sum  s   
  r  c                 C   sN   |dur%|j d dkr%|j d dkr%| j}| |dddddf  |} | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    Nr   r   )ri   r=   ro   )r   r   r=   r4   r4   r5   apply_mask_to_padding_states  s   $ r  c                       s   e Zd ZdZdedef fddZ				ddejde	e
 d	e	ej d
e	ej de	ej f
ddZ			dde	e
 d	e	ej d
e	ej fddZ				dde	e
 d	e	ej d
e	ej de	ej fddZ  ZS )
BambaMixeruP  
    Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)

    The are a few differences between this and Mamba2Mixer:
    - The variable use_precomputed_states is slightly different due to the hybrid cache structure
    - There's a few non-obvious bugs fixed with batching in the slow path that exist in main
    - Some extra variables that our layer doesn't need have been removed
    - We ported most of the refactors in https://github.com/huggingface/transformers/pull/35154, which is (as of Dec 18, 2024) unmerged
    r8   rc   c                    s  t    |j| _|j| _|j| _|j| _t	|j
| j | _|| _|j| _|j| _t|j | _|j| _|j| _|j| _|j| _|j| _dtdf| _d| _d| _ | jd| j | j  | _!t"j#| j!| j!|j| j| j!| jd d| _$| j| j! | j }t"j%| j|| jd| _&t"'t()| j| _*t(+d| jd }t"'t(,|| _-t.| j| jd	| _/t"'t()| j| _0t"j%| j| j| jd| _1t2st34d
 d S t34d d S )Nr   r  gMbP?g?r:   r   )in_channelsout_channelsr   kernel_sizegroupspaddingr   r   a  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1dzDThe fast path for Bamba will be used when running the model on a GPU)5r   r`   rW   	num_headsrU   rM   r^   rL   r]   r2   rT   intermediate_sizerc   mamba_conv_biasuse_conv_bias
hidden_act
activationr   actmamba_proj_biasuse_biasrms_norm_epslayer_norm_epsilonrV   n_groupsrX   r   mamba_chunk_sizer  r   time_step_limittime_step_mintime_step_maxconv_dimr   Conv1dconv1dr   in_projr   r/   r   dt_biasarangelogA_logr   normDout_projis_fast_path_availableloggerwarning_once)r\   r8   rc   projection_sizeAr   r4   r5   r`     sX   

	zBambaMixer.__init__Nr   cache_paramsr   r   r*   c                 C   s  t ||}| |}|j\}}}	| j| j }
|d uoD|joD|dkoD|j| j jd |j| j jd   ko8|kn  oD|d uoD|d dk}|r)|	dj
| j| j| jgdd\}}}t||j| j | jj	d| jj| j}tj
|| j|
|
gdd\}}}t| j  }|d d d df d d d d d f d| j| jjtjd}|d d d d d f dd| j}| jd d d df d| j}| jd d d df d| j}||| j|jd | j }||| j|jd | j }||| j| j}t|j| j ||||||d |dd
}||| j| j }| ||}|  |d d d df }|S t| j  }| j!d	td
fkr>i nd| j!i}| j"r||d u r|t#|| jj	d| jj| j|f| j| j$|| j| jj| jj%| j j| j j| j| jddd|}|S |j
| j| j| jgdd\}}}|d ur|&dd}t'j()|| j*|jd  df}|j| j +| | jdvr| ,| |&dddd |f &dd}nt-|&dd| jj	d| jj| j|d&dd}t ||}tj
|| j|
|
gdd\}}}t.|||d| j|||||| jd|||| jdf| j$| jd |d| jdd|\}}|d ur:|d ur:|j| j +| |||d}| ||}|  |}|S )Nr   r   rf   rg   .r   T)zr0  dt_softplusr   r  dt_limitF)r5  r  r*   r!  rmsnorm_weightrmsnorm_epsoutproj_weightoutproj_biasheaddimngroupsnorm_before_gatereturn_final_statesr:   )r   swish)r   r   r   r!  r*   )r  r5  r=  r*   rG  r0  r>  )/r  r/  ri   r'  r^   rK   rN   rc   rO   squeezesplitr  r,  r  r#   r.  r   r   r!  r/   expr3  r   r   r   ro   r   r0  r5  r   r   r4  r6  r)  r   r!   r  r   r   r   r   r  r]   copy_r"  r"   r    )r\   r   r<  r   r   r*   projected_statesrG   seq_lenrE   groups_time_state_sizeuse_precomputed_statesr   hidden_states_B_CdtBCr;  r0  r5  hidden_states_reshapedoutdt_limit_kwargshidden_states_B_C_transposedrN   scan_output	ssm_stater4   r4   r5   cuda_kernels_forward  s  
	




<"
^"V
$




zBambaMixer.cuda_kernels_forwardc           3   
      s  |j \}}}|j}t||}|}	|	jjjjgdd\}
}}|d uoQ|joQ|dkoQ|j	j
 j d |jj
 j d   koE|kn  oQ|d uoQ|d dk}|r|j	j
 jddd|j	j
< |d d dd d f |j	j
 j|j	j
 d d d d df< |j	j
 jjjjd}tj|jjd dd}jr|jj }|}n8|d ur|dd}tj|j|j d  df}|j	j
 | |dddd |f dd}t||}tj|jjj jj gdd\}}}tj !  }|r[|jj
 j}|d d dd d f d d d df }|dd"||j d j#}j$d	 "j$j d j#}tjj%|||j }t&|j'd j'd }|d
 "jj#jjtj(d}t|d	 | j|d}|)|jddd d d f }|"|jjj |j d * }|)|d|j d }|d	 |dd d d f  }|)|dj#}||d	  j|d}|jj
 |jj
 | |  |)|jddd d d f }|"|jjj |j d * }|)|d|j d }|jj
 j|j|jd}|+|j j#j}|+|j jd}t,||}|+|jj#}j-d	 "j-j d j#}|||  |j}|)|dd d d df }ntj%|j$ }t&|j'd j'd }|)||dj#! }|)||dj! }|)||dj! }|j.jj djd}|j.jj djd}j/|j/  j/  j-d	 t0|  }||d	  }||j| } fdd||||fD \}}}}|1dddd}tj2|dd}tt3|} |d d d d d d d d d d d f |d d d d d d d d d d d f  }!|!jdd}"|"d	 | 1dddddd	  }#|#jdd}$|$d	 |d d d d d f  jdd}%t|d d d d d d dd f | }&||&1ddddd	  }'|'dd d d f |d	  jdd}(|r|jj
 d d d df j|(jd})nt4|(d d d df })tj5|)|(gdd}(tt3tj|d d d d d d df d}*|*dd}*|*d
 |(d d d d d df  jdd}+|+d d d df |+d d df }(},t|}-|dd d d f |(d d d d d df  }.|-1dddd}/|.d|/d	  }0|%|0 }|)|djj#}|| } dkrB|d d d |d d d d f }|)||d}|,d ur\|d ur\|jj
 |, 6||
}17|1|}2|2S )Nrf   rg   r   r   )shiftsdimsr>   r:   .r  ).NNr   r;   )rh   output_sizec                    s   g | ]	}t | jqS r4   )r  r  )rD   tr   r\   r4   r5   rH   P  s    z,BambaMixer.torch_forward.<locals>.<listcomp>r	   r   rq   )r   r   )8ri   r=   r  r/  rJ  r  r,  r  rK   rN   rc   rO   rollro   r<   r.  r   r/   sumrI  r  r   r"  r   r   r   r  r]   rL  r'  r^   rK  r3  r   r   r   r0  softplusclampr)  r   r   r   r   bmmr5  repeat_interleaver  r  permuter  r  
zeros_likerj   r4  r6  )3r\   input_statesr<  r   r   rG   rN  rE   r=   rM  r   rQ  rR  rP  rN   rX  r   rS  rT  r;  cache_devicer0  dAdBdBxrO   ssm_states_reshaped
C_reshapedyr5  
D_residualA_cumsumLG_intermediateGM_intermediateMY_diagdecay_statesB_decaystatesprevious_statesdecay_chunk
new_statesrZ  state_decay_outC_times_statesstate_decay_out_permutedY_offrY  contextualized_statesr4   r`  r5   torch_forward  s   


@,
$"$$$P&*"&0(&
*
 zBambaMixer.torch_forwardc                 K   s   t rd| jjjjv r| |||||S |d urtd|j}|d ur@|jd dkr@|jd dkr@||d d d d d f  	|}| 
||||S )Ncudaz\`seq_idx` support requires fast path support. Please install `mamba_ssm` and `causal_conv1d`r   r   )r7  r/  r   r<   r~   r[  NotImplementedErrorr=   ri   ro   r  )r\   r   r<  r   r   r*   r   r=   r4   r4   r5   r     s   	$ zBambaMixer.forward)NNNN)NNN)r+   r,   r-   r.   r   r2   r`   r/   rv   r   r7   r0   r3   r[  r  r   r   r4   r4   r   r5   r    sV    D
 .
 Sr  c                       s$   e Zd Z fddZdd Z  ZS )BambaMLPc                    sx   t    || _|j| _|j| _tj| j| j|jd| _tj| j| j|jd| _	tj| j| j|jd| _
t|j | _d S )Nr   )r   r`   r8   rU   r  r   r   mlp_bias	gate_projup_proj	down_projr   r   act_fnr\   r8   r   r4   r5   r`     s   
zBambaMLP.__init__c                 C   s$   |  | | || | }|S rs   )r  r  r  r  )r\   r   r  r4   r4   r5   r     s    zBambaMLP.forwardr   r4   r4   r   r5   r    s    
r  RMSNormc                       s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	BambaRMSNormr   c                    s&   t    tt|| _|| _dS )z;
        BambaRMSNorm is equivalent to T5LayerNorm
        Nr   r   r   r4   r5   r`     s   

zBambaRMSNorm.__init__c                 C   sJ   |j }|tj}|djddd}|t|| j  }| j|| S r   )	r=   ro   r/   r   r   r   r   r   r   )r\   r   r   r   r4   r4   r5   r     s
   zBambaRMSNorm.forwardc                 C   s   t | jj d| j S )Nz, eps=)ry   r   ri   r   )r\   r4   r4   r5   
extra_repr  s   zBambaRMSNorm.extra_reprr   )r+   r,   r-   r`   r   r  r   r4   r4   r   r5   r    s    r  c                       s   e Zd Zddededef fddZeddd	d
							ddej	de
ej	 de
ej de
e de
e de
e de
ej de
eej	ej	f  dee deeje
eejejf  f fddZ  ZS )BambaDecoderLayerr9   r8   rc   
layer_typec                    s   t    d}|dkrtnd }||| _t|j|jd| _t|j|jd| _|| _	|dkr6t
||d| _d S |dkrBt||| _d S td)Nr   r  r9   )r8   rc   	attentionzInvalid layer_type)r   r`   r  feed_forwardr  rU   r%  input_layernormpre_ff_layernormr  r  r9   r   	self_attn
ValueError)r\   r8   rc   r  num_expertsffn_layer_classr   r4   r5   r`     s   

zBambaDecoderLayer.__init__r   r   r   r   NFr   r   r   output_attentions	use_cacher   r   r   re   c	                 K   s   |}
|  |}| jdkr| jd||||d|	}d}n| jdkr4| jd||||||||d|	\}}|
| }|}
| |}| |}|
| }|f}|rR||f7 }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`HybridMambaAttentionDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence.
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs. Can be used to provide `BambaFlashAttentionKwargs` for
                padding-free training and/or improve torch.compile performance.
        r9   )r   r<  r   r   Nr  )r   r   r   r   r  r  r   r   r4   )r  r  r9   r  r  r  )r\   r   r   r   r   r  r  r   r   r   residualself_attn_weightsoutputsr4   r4   r5   r     sD   #


	



zBambaDecoderLayer.forward)r9   )NNNFFNN)r+   r,   r-   r   r2   rx   r`   r   r/   rv   r   r0   r7   r  ry   r   r%   FloatTensorr   r   r4   r4   r   r5   r    s>    	
r  c                       sD   e Zd ZU eed< dZdZdgZdZdZ	dZ
dZ fddZ  ZS )BambaPreTrainedModelr8   modelTr  r   c                    sV   t  | t|tr)|jjd tt	d|j
d |j_|jjd d S d S )Ng      ?r   )r   _init_weightsr   r  r0  datafill_r/   r2  r1  r  r3  r5  )r\   r   r   r4   r5   r  ?  s   
z"BambaPreTrainedModel._init_weights)r+   r,   r-   r   r1   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_is_statefulr  r   r4   r4   r   r5   r  3  s   
 r  c                       s   e Zd Zdef fddZee									ddeej	 deej
 deej	 dee d	eej d
ee dee dee deej	 dee defddZdej
dej
dej
dedef
ddZedej
dededejdej
defddZdd Z  ZS )
BambaModelr8   c                    s   t  | |j| _|j| _t|j|j| j| _g }t	|j
D ]}|t|||j| d q t|| _|j| _t|j|jd| _t|d| _d| _|   d S )N)rc   r  r  )r8   F)r   r`   pad_token_idpadding_idx
vocab_sizer   	EmbeddingrU   embed_tokensrQ   rR   rY   r  rJ   
ModuleListlayersr   r  r%  final_layernormrz   
rotary_embgradient_checkpointing	post_init)r\   r8   decoder_layersr_   r   r4   r5   r`   I  s   zBambaModel.__init__N	input_idsr   r   r   inputs_embedsr  r  output_hidden_statesr   r   re   c
                 K   s  |d ur|n| j j}|d ur|n| j j}|d ur|n| j j}|d u |d uA r*td| jr9| jr9|r9td d}|d u rB| 	|}|}|rO|d u rOtd |	d u r^t
j|jd |jd}	|d u rg|	d}| |||	||}| ||	}| ||}|rdnd }|rdnd }| jD ]5}|jd	kr|n|}|r||f7 }||f||||||	|d
|
}|d }|r|d d ur||d f7 }q| |}|r||f7 }|r|jsd|_|sd n|}t||||dS )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzBamba requires an initialized `HybridMambaAttentionDynamicCache` to return a cache. None was provided, so no cache will be returned.r   r>   r   r4   r9   )r   r   r   r  r  r   r   T)last_hidden_stater   r   
attentions)r8   r  r  r  r  r  r   r8  r9  r  r/   r1  ri   r<   r   _update_causal_mask_update_mamba_maskr  r  r  r  rK   r   )r\   r  r   r   r   r  r  r  r  r   r   r   r   
mamba_maskr   all_hidden_statesall_self_attnsdecoder_layer
layer_masklayer_outputs
next_cacher4   r4   r5   r   \  s~   




	


zBambaModel.forwardr   c                 C   s   | j jdkr|d urd|v r|S d S |d ur| nd}| j jdkr0|s0tj|||| jdr0d S |j}|jd }t|t	j
rC|jd n|| d }	| j|||	|||jd d}
| j jdkru|d uru|jjd	v ru|sut	|j}t|
|}
|
S )
Nflash_attention_2r   r   sdpa)r  past_key_values_lengthis_trainingr   rf   )sequence_lengthtarget_lengthr=   r   rG   )r  xpunpu)r8   r   rr   r   _ignore_causal_mask_sdpar   r=   ri   r   r/   rv   5_prepare_4d_causal_attention_mask_with_cache_positionr<   r~   finfomin_unmask_unattended)r\   r   r   r   r   r  past_seen_tokensr=   r  r  r   	min_dtyper4   r4   r5   r    sF   



zBambaModel._update_causal_maskr  r  r=   rG   c                 K   s|  | dur|   dkr| }|S t|j}tj||f|||jd}|dkr+tj|dd}|tj||jd|ddk9 }|ddddddf 	|ddd}| dur|
 }| jd }	| ddddddf | ddddddf kdddd| dddf |}
|ddddddd|	f |
 }|dk}|ddddddd|	f |||ddddddd|	f< |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        Nr   )
fill_valuer=   r<   r   r	  r>   rf   r   )rh   r/   r  r  fullr<   triur1  r   r   cloneri   ro   r  )r   r  r  r=   r   rG   r   r   r  mask_lengthpadding_attention_maskpadding_maskr4   r4   r5   r    s2    $
.$  z@BambaModel._prepare_4d_causal_attention_mask_with_cache_positionc                 C   s.   |}|d dks|durt |dkrd}|S )zv
        No need for zeroing states when
            1. Cached forward
            2. Attending to all inputs
        r   Nr   )r/   all)r\   r   r   r  r4   r4   r5   r  4  s   "zBambaModel._update_mamba_mask)	NNNNNNNNN)r+   r,   r-   r   r`   r   r   r   r/   r0   rv   r7   r  r  r   r%   r   r   r  staticmethodr2   r=   r  r  r   r4   r4   r   r5   r  G  s|    	
b
<7r  c                       s   e Zd ZdgZddiZddgdgfiZ fddZee												dd
e	e
j de	e
j de	e
j de	e de	e
j de	e
j de	e de	e de	e de	e
j deee
jf defddZ						dddZ  ZS )BambaForCausalLMzlm_head.weightlm_headcolwise_repr   logitsc                    sH   t  | t|| _|j| _tj|j|jdd| _|j	| _	| 
  d S )NFr   )r   r`   r  r  r  r   r   rU   r  z_loss_coefficientr  r  r   r4   r5   r`   F  s   
zBambaForCausalLM.__init__Nr   r  r   r   r   r  labelsr  r  r  r   logits_to_keepre   c                 K   s   |dur|n| j j}|	dur|	n| j j}	| jd
||||||||	|
d	|}|j}t|tr4t| dn|}| |dd|ddf }d}|durt| j	d
||| j j
d|}| jdkrt|jddj|jdd }|| j|  }t|||j|j|jd	S )aJ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, BambaForCausalLM

        >>> model = BambaForCausalLM.from_pretrained("...")
        >>> tokenizer = AutoTokenizer.from_pretrained("...")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)	r  r   r   r   r  r  r  r  r   )r  r  r  r   rf   rg   r   r:   )lossr  r   r   r  r4   )r8   r  r  r  r  r   r2   slicer  loss_functionr  r  	logsumexpro   r=   r   r   r   r   r   r  )r\   r  r   r   r   r  r  r  r  r  r   r  r   r  r   slice_indicesr  r  z_lossr4   r4   r5   r   P  s@   '

 zBambaForCausalLM.forwardTc              	   K   s<  |d u }	|	s5|d us|d |j d kr"|d d |j d  d f }n!|j d |j d kr4|d d |f }nt| j|j d | j| jd}|d url|d u rl| dd }||dkd |	sl|d d |j d  d f }|d urw|	rwd|i}
nd| i}
|
	||||| jj
|d | D ]\}}||
vr||
|< q|
S )Nrf   r   r   r>   r  r  )r   r   r  r   r  r   )ri   r7   r8   r=   r<   longr  masked_fill_r   rk   num_logits_to_keepitems)r\   r  r   r   r  r   r   r  r   empty_past_kvmodel_inputsr   r   r4   r4   r5   prepare_inputs_for_generation  sB   
z.BambaForCausalLM.prepare_inputs_for_generation)NNNNNNNNNNr   )NNNNNT)r+   r,   r-   _tied_weights_keys_tp_plan_pp_planr`   r   r   r   r/   r0   rv   r7   r  r  r   r2   r   r   r  r   r4   r4   r   r5   r  @  sd    
	
Pr  )r  r  r  )r   )Nr   )Qtypingr   r   r   r   r   r/   r   transformers.activationsr   cache_utilsr
   
generationr   integrationsr   modeling_attn_mask_utilsr   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.deprecationr   utils.import_utilsr   r   configuration_bambar   +mamba_ssm.ops.triton.selective_state_updater   !mamba_ssm.ops.triton.ssd_combinedr    r!   causal_conv1dr"   r#   
get_loggerr+   r8  r%   r7   Modulerz   r   rv   r2   r   r   r   r   r   r   r  r  r  r  r7  r  r  r  r  r  r  r  r  __all__r4   r4   r4   r5   <module>   s   
]$

(G   ba y  