o ei@sddlmZddlmZmZmZddlZddlmZddlm Z ddl m Z ddl mZdd lmZdd lmZmZdd lmZdd lmZdd lmZddlmZmZddlmZmZddl m!Z!m"Z"ddl#m$Z$ddl%m&Z&m'Z'm(Z(m)Z)m*Z*ddl+m,Z,ddl-m.Z.e*/e0Z1GdddeddZ2GdddZ3Gdddej4Z5ddZ6d ej7d!e8d"ej7fd#d$Z9 %dPd&ej4d'ej7d(ej7d)ej7d*ej7dBd+e:d,e:d-e$e&fd.d/Z;dQd0d1Zd6ej7d7e8fd8d9Z?d:d;Z@dd?ZBGd@dAdAej4ZCGdBdCdCej4ZDedDGdEdFdFej4ZEGdGdHdHeZFe'GdIdJdJe"ZGe'GdKdLdLeGZHe'GdMdNdNeGeZIgdOZJdS)R)Callable)AnyOptional TypedDictN)nn)ACT2FN)initialization)Cache)GenerationMixin)use_kernel_forward_from_hubuse_kernelized_func)lazy_load_kernel)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torchdynamo_compilinglogging)maybe_autocast) BambaConfigc@s@eZdZUdZejed<ejed<eed<eed<ejed<dS)BambaFlashAttentionKwargsaU Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage. Use cases include padding-free training and fewer `torch.compile` graph breaks. cu_seq_lens_q (`torch.LongTensor`): Gets cumulative sequence length for query state. cu_seq_lens_k (`torch.LongTensor`): Gets cumulative sequence length for key state. max_length_q (`int`): Maximum sequence length for query state. max_length_k (`int`): Maximum sequence length for key state. seq_idx (`torch.IntTensor`): Index of each packed sequence. cu_seq_lens_q cu_seq_lens_k max_length_q max_length_kseq_idxN) __name__ __module__ __qualname____doc__torch LongTensor__annotations__int IntTensorr/r/f/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/bamba/modeling_bamba.pyr 5s   r F)totalc @seZdZdZdZejdfdefddZddZ d d Z dd ej d ej d e de eefdBdeej ej ff ddZdejfddZdej d e dee e ffddZdd e dBde fddZdS) HybridMambaAttentionDynamicCachea A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache (which has a constant shape regardless of seq_len). This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states` and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`, while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors). For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors), while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`, and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`. FNconfigc s0|j|_d|_|j}|j}g|_g|_g|_t|jD]^}|j|dkrS|jt j |j |j d|j |||dg7_|jt j |j|j||dg7_q|jt jggdg7_|jt jggdg7_|j|qfddt|jD|_fddt|jD|_dS) NFmambadevicedtyper7c g|] }tjggdqSr9r*tensor.0_ batch_sizer7r/r0  z=HybridMambaAttentionDynamicCache.__init__..cr:r;r<r>rAr/r0rCrD)layers_block_typehas_previous_state mamba_d_conv mamba_d_state conv_states ssm_statestransformer_layersrangenum_hidden_layersr*zeros mamba_expand hidden_sizemamba_n_groups mamba_n_heads mamba_d_headr=append key_cache value_cache)selfr3rBr8r7conv_kernel_sizessm_state_sizeir/rAr0__init__]sB   z)HybridMambaAttentionDynamicCache.__init__cCs t|jSN)lenrUrWr/r/r0__len__s z(HybridMambaAttentionDynamicCache.__len__cCs|j||j|fSr\)rUrVrW layer_idxr/r/r0 __getitem__sz,HybridMambaAttentionDynamicCache.__getitem__ key_states value_statesra cache_kwargsreturncCsz|j|jddkr||j|<||j|<ntj|j||gdd|j|<tj|j||gdd|j|<|j||j|fS)Nrr5dim)rUshaperVr*cat)rWrcrdrarer/r/r0updates   z'HybridMambaAttentionDynamicCache.updatebeam_idxcCs|dkrdtt|jD]X}|j|j}|j|d|||j|<|j|j}|j|d|||j|<|j|j}|j|d|||j|<|j |j}|j |d|||j |<q dSdS)zDReorders the cache for beam search, given the selected beam indices.rN) get_seq_lengthrLr]rUr7 index_selecttorVrIrJ)rWrmrar7r/r/r0 reorder_caches      z.HybridMambaAttentionDynamicCache.reorder_cachecache_positioncCs$d}|jd}|||}||fS)zDReturn the length and offset of the cache, used to generate the maskr)rjrn)rWrrra kv_offset query_length kv_lengthr/r/r0get_mask_sizess z/HybridMambaAttentionDynamicCache.get_mask_sizesrcCsN||jvr |jdn|}t|j|ks|j|jddkrdS|j|jdS)zYReturns the sequence length of the cached states. A layer index can be optionally passed.rrg)rKr]rUrjr`r/r/r0rns"z/HybridMambaAttentionDynamicCache.get_seq_lengthr\)r)r&r'r(r)is_compileabler*float16rr[r_rbTensorr-dictstrrtuplerlr+rqrvrnr/r/r/r0r2Ms* &  r2c s~eZdZUejed<ddeffdd Ze   ddedBde dde dBd e d e ffd d Z eed dZZS)BambaRotaryEmbeddinginv_freqNr3cst|j|_|j|_||_|jjd|_|j}|jdkr$t |j}||j|\}|_ |j d|dd|j d| dddS)N rope_typedefaultrF) persistentoriginal_inv_freq) superr[max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr3rope_parametersrcompute_default_rope_parametersrattention_scalingregister_bufferclone)rWr3r7 rope_init_fnr __class__r/r0r[s   zBambaRotaryEmbedding.__init__r7z torch.deviceseq_lenrfz torch.TensorcCsZ|jd}t|ddp|j|j}d}d|tjd|dtjdj|tjd|}||fS) a Computes the inverse frequencies according to the original RoPE implementation Args: config ([`~transformers.PreTrainedConfig`]): The model configuration. device (`torch.device`): The device to use for initialization of the inverse frequencies. seq_len (`int`, *optional*): The current sequence length. Unused for this type of RoPE. Returns: Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). rope_thetahead_dimNg?rr5r8r6) rgetattrrPnum_attention_headsr*arangeint64rpfloat)r3r7rbaseriattention_factorrr/r/r0rs &z4BambaRotaryEmbedding.compute_default_rope_parametersc Cs|jddddf|jddd|j}|dddddf}t|jjtr6|jjdkr6|jjnd}t |dd+|| dd}t j ||fdd }| |j}||j} Wdn1slwY|j|jd | j|jd fS) NrrgrmpscpuF) device_typeenabledr5rhr)rrexpandrjrpr7 isinstancetyper|r transposer*rkcosrsinr8) rWx position_idsinv_freq_expandedposition_ids_expandedrfreqsembrrr/r/r0forwards0&zBambaRotaryEmbedding.forwardr\NNN)r&r'r(r*rzr,rr[ staticmethodrr-r}rrno_gradrr __classcell__r/r/rr0r~s&   r~cCsH|dd|jddf}|d|jdddf}tj| |fddS)z*Rotates half the hidden dims of the input..Nrgr5rh)rjr*rk)rx1x2r/r/r0 rotate_halfsr hidden_statesn_reprfcCs^|j\}}}}|dkr |S|dddddddddf|||||}||||||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)rjrreshape)rrbatchnum_key_value_headsslenrr/r/r0 repeat_kvs 0rmodulequerykeyvalueattention_maskscalingdropoutkwargsc Kst||j}t||j} t||dd|} |dur | |} tjj| dtjd |j } tjj | ||j d} t| | } | dd } | | fS)Nr5rrg)rir8)ptrainingr)rnum_key_value_groupsr*matmulrr functionalsoftmaxfloat32rpr8rr contiguous) rrrrrrrrrcrd attn_weights attn_outputr/r/r0eager_attention_forward s  rc Cs||}||}|jd}|dd|f|d|df}}|dd|f|d|df}} ||t||} ||t||} tj| |gdd} tj| | gdd} | | fS)aApplies Rotary Position Embedding to the query and key tensors. Removes the interleaving of cos and sin from GLM Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. rg.Nrh) unsqueezerjrr*rk) qkrr unsqueeze_dim rotary_dimq_rotq_passk_rotk_passq_embedk_embedr/r/r0apply_rotary_pos_emb&s   ""rcseZdZdZdedeffdd Z    ddejde ejejfdBd ejdBd e dBd ej dBd e e d e ejejffddZZS)BambaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr3racst||_||_t|d|j|j|_|j|j|_ |jd|_ |j |_ d|_ t j|j|j|j|jd|_t j|j|j|j|jd|_t j|j|j|j|jd|_t j|j|j|j|jd|_dS)NrgTbias)rr[r3rarrPrrrrrattention_dropout is_causalrLinearattention_biasq_projk_projv_projo_proj)rWr3rarr/r0r[Ps(   zBambaAttention.__init__Nrposition_embeddingsrpast_key_valuesrrrrfcKs|jdd}g|d|jR}|||dd} |||dd} |||dd} |\} } t| | | | \} } |durW| | |d}|| | |j |\} } t |j j t}||| | | |f|jskdn|j|jd|\}}|jg|dR}||}||fS)Nrgrr5)rrrrr)rr)rjrrviewrrrrrlrar get_interfacer3_attn_implementationrrrrrrr)rWrrrrrrr input_shape hidden_shape query_statesrcrdrrreattention_interfacerrr/r/r0rgs8    zBambaAttention.forwardNNNN)r&r'r(r)rr-r[r*rzr}r r+rrrrr/r/rr0rLs,rcs(eZdZdfdd ZdddZZS) BambaRMSNormGatedư>cs&ttt||_||_dSr\rr[r Parameterr*onesweightvariance_epsilonrWrPepsrr/r0r[s  zBambaRMSNormGated.__init__NcCsj|j}|tj}|dur|tj|tj}|djddd}|t ||j }|j ||SNr5rgT)keepdim) r8rpr*rrrsilupowmeanrsqrtrr)rWrgate input_dtypevariancer/r/r0rs zBambaRMSNormGated.forwardrr\r&r'r(r[rrr/r/rr0rsr input_tensorpad_sizecCsHt|jdkrddddd|ddfnddd|ddf}tjjj||dddS)z Padding x tensor with `pad_size` on the seq_len dim (dim=1) Assumes that we only have tensors of either size 4 or 3 rconstant)moder)r]rjr*rrpad)rr pad_shaper/r/r0pad_tensor_by_sizes2r cCsXt||}t|jdkr||jdd||jdS||jdd||jd|jdS)z Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and simultaneously splitting it into chunk sequences. Assumes that we only have tensors of either size 4 or 3 rrrgr5)r r]rjr)rr chunk_sizer/r/r0reshape_into_chunkss r cCs|d}|djg||R}tjtj|||jtjddd}||d}tj|dd}tjtj|||jtjddd}||tj }|S)zo More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions. rg.Nr6)diagonalrrwrh) sizerr*trilrr7bool masked_fillcumsuminf)rr mask tensor_segsumr/r/r0 segment_sums   rcCsN|dur%|jddkr%|jddkr%|j}||dddddf|}|S)zm Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66 Nrr)rjr8rp)rrr8r/r/r0apply_mask_to_padding_statess$ rc seZdZdZdedeffdd Z    ddejde dBd ej dBd ejdBd ej dBf d d Z   dde dBd ej dBd ejdBfddZ    dde dBd ej dBd ejdBd ej dBfddZZS) BambaMixeruP Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`. A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective) ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4, and is why Mamba is called **selective** state spaces) The are a few differences between this and Mamba2Mixer: - The variable use_precomputed_states is slightly different due to the hybrid cache structure - There's a few non-obvious bugs fixed with batching in the slow path that exist in main - Some extra variables that our layer doesn't need have been removed - We ported most of the refactors in https://github.com/huggingface/transformers/pull/35154, which is (as of Dec 18, 2024) unmerged r3racst|j|_|j|_|j|_|j|_t |j |j|_ ||_ |j |_|j|_t|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j d|j|j|_ t!j"|j |j |j |j|j |jdd|_#|j |j |j}t!j$|j||jd|_%t!&t'(|j|_)t'*d|jd}t!&t'+||_,t-|j |jd|_.t!&t'(|j|_/t!j$|j |j|jd|_0t1d}t2|dda3t2|dda4t1d }t2|d da5t2|d da6t2|d da7t8t5t4t3fa9t9st:;d dSt:;ddS)Nr5r) in_channels out_channelsr kernel_sizegroupspaddingrrz causal-conv1dcausal_conv1d_updatecausal_conv1d_fnz mamba-ssmselective_state_updatemamba_chunk_scan_combined mamba_split_conv1d_scan_combinedaThe fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1dzDThe fast path for Bamba will be used when running the model on a GPU)ind |j!i}|j"r||dur|t#||jj d|jj|j|f|j|j$||j|jj|jj%|j j|j j|j|jd d d |}|S|j |j |j |j gdd\} } }|dur| &dd}t'j()||j*|jddf}|j|j+||jdvr|,|| &dddd|f&dd} nt-| &dd|jj d|jj|j|d&dd} t| |} tj | |j | | gdd\}}}t.|||d|j||||||jd||||jdf|j$|jd|d|jdd|\}}|dur:|dur:|j|j+||||d}||| }| |}|S)Nrrrgrh.rT)zr9 dt_softplusrrdt_limitF) r=r r%r*rmsnorm_weight rmsnorm_epsoutproj_weight outproj_biasheaddimngroupsnorm_before_gatereturn_final_statesr5)rswish)rrrr*r%)r r=rHr%rRr9rI)/rr8rjr0rYrFrIrarJsqueezesplitr&r5r%r r7rrr*r*expr;rrrrprr9r=rr"r<r>r2rr$r rrrrrrXcopy_r+r!r#)rWrrGrrrr%projected_statesrBrr@groups_time_state_sizeuse_precomputed_statesrhidden_states_B_CdtBCrDr9r=hidden_states_reshapedoutdt_limit_kwargshidden_states_B_C_transposedrI scan_output ssm_stater/r/r0cuda_kernels_forwardDs          <"  ^" V   $        zBambaMixer.cuda_kernels_forwardc3 s|j\}}}|j}t||}|} | jjjjgdd\} } } |duoQ|joQ|dkoQ|j j jd|j j jdkoE|knoQ|duoQ|ddk} | r|j j j ddd|j j <| dddddf |j j j|j j dddddf<|j j j jjjd}tj|jjddd} jr| jj} | } n8|dur| dd}tj|j|jddf}|j j || dddd|fdd} t| |} tj| jjjjjgdd\}}}tj ! }| r[|j j j}| dddddfddddf} | dd"|| jdj#} j$d "j$jdj#}tjj%| | | j} t&| j'dj'd} |d "jj#jj tj(d }t| d |j |d}|)|jdddddf}|"|jjj|jd*}|)|d|jd}| d |ddddf}|)|dj#}||d j |d}|j j |j j |||)|jdddddf}|"|jjj|jd*}|)|d|jd}|j j j |j|jd }|+|jj#j}|+|jjd}t,||}|+|jj#}j-d "j-jdj#}||| |j}|)|dddddf}ntj%| j$} t&| j'dj'd} |)||dj#!}|)||dj!}|)||dj!}|j.jjdjd }|j.jjdjd }j/|j/j/j-d t0|}|| d }| |j| }fdd||||fD\}}}}|1dddd}tj2|dd}tt3|} |dddddddddddf|dddddddddddf}!|!jdd}"|"d | 1dddddd }#|#jdd}$|$d |dddddfjdd}%t|ddddddddf|}&||&1ddddd }'|'ddddf|d jdd}(| r|j j ddddfj |(jd})n t4|(ddddf})tj5|)|(gdd}(tt3tj|dddddddfd}*|*dd}*|*d |(ddddddfjdd}+|+ddddf|+dddf}(},t|}-|ddddf|(ddddddf}.|-1dddd}/|.d|/d }0|%|0}|)|djj#}||}dkrB|ddd|ddddf}|)||d}|,dur\|dur\|j j |,6|| }17|1 |}2|2S)Nrgrhrr)shiftsdimsr9r5.r ).NNrr6)ri output_sizecsg|] }t|jqSr/)r r )r?trrWr/r0rCwsz,BambaMixer.torch_forward..rrrw)rr)8rjr8rr8rUr&r5r%rFrIrarJrollrpr7r7rr*sumrTr(rr+rrrrrXrWr0rYrVr;rrrr9softplusclampr2rrrrbmmr=repeat_interleaver r permuterr zeros_likerkr<r>)3rW input_statesrGrrrrBrr@r8rXrr[r\rZrIrbrr]r^rD cache_devicer9dAdBdBxrJssm_states_reshaped C_reshapedyr= D_residualA_cumsumLG_intermediateGM_intermediateMY_diag decay_statesB_decaystatesprevious_states decay_chunk new_statesrdstate_decay_outC_times_statesstate_decay_out_permutedY_offrccontextualized_statesr/rjr0 torch_forwards      @   ,  $"$ $  $P  &*"&0 (& *   zBambaMixer.torch_forwardcKstrd|jjjjvrts||||||S|durtd|j}|durC|j ddkrC|j ddkrC||dddddf |}| ||||S)Ncudaz\`seq_idx` support requires fast path support. Please install `mamba_ssm` and `causal_conv1d`rr) r@r8rr7rrreNotImplementedErrorr8rjrpr)rWrrGrrrr%rr8r/r/r0rs $ zBambaMixer.forwardrr)r&r'r(r)rr-r[r*rzr2r+r.rerrrr/r/rr0rsV Q . Srcs$eZdZfddZddZZS)BambaMLPcsxt||_|j|_|j|_tj|j|j|jd|_tj|j|j|jd|_ tj|j|j|jd|_ t |j |_ dS)Nr)rr[r3rPr&rrmlp_bias gate_projup_proj down_projrr)act_fnrWr3rr/r0r[s zBambaMLP.__init__cCs$||||||}|Sr\)rrrr)rWrrr/r/r0rs zBambaMLP.forwardrr/r/rr0rs  rRMSNormcsFeZdZd deddffdd Zdejdejfdd Zd d ZZ S) BambaRMSNormrrrfNcs&ttt||_||_dS)z; BambaRMSNorm is equivalent to T5LayerNorm Nrrrr/r0r[s  zBambaRMSNorm.__init__rcCsJ|j}|tj}|djddd}|t||j}|j||Sr) r8rpr*rrrrrr)rWrrrr/r/r0rs  zBambaRMSNorm.forwardcCst|jjd|jS)Nz, eps=)r}rrjrr^r/r/r0 extra_reprszBambaRMSNorm.extra_reprr) r&r'r(rr[r*rzrrrr/r/rr0rsrcseZdZddededeffdd Z       dd ejd ejdBd ej dBd e dBd e dBde dBdej dBde ejejfdBde ede eje ejejfdBffddZZS)BambaDecoderLayerr4r3ra layer_typecstd}|dkr tnd}|||_t|j|jd|_t|j|jd|_||_ |dkr6t ||d|_ dS|dkrBt |||_ dStd)Nrrr4)r3ra attentionzInvalid layer_type)rr[r feed_forwardrrPr.input_layernormpre_ff_layernormrrr4r self_attn ValueError)rWr3rar num_expertsffn_layer_classrr/r0r[s  zBambaDecoderLayer.__init__NFrrrroutput_attentions use_cacherrrrrfc Ks|} ||}|jdkr|jd||||d| }d} n|jdkr4|jd||||||||d| \}} | |}|} ||}||}| |}|f} |rR| | f7} | S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, sequence_length)` where padding elements are indicated by 0. past_key_values (`HybridMambaAttentionDynamicCache`, *optional*): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. kwargs (`dict`, *optional*): Arbitrary kwargs. Can be used to provide `BambaFlashAttentionKwargs` for padding-free training and/or improve torch.compile performance. r4)rrGrrrNr)rrrrrrrrrr/)rrr4rrr) rWrrrrrrrrrrresidualself_attn_weightsoutputsr/r/r0r sD"        zBambaDecoderLayer.forward)r4)NNNFFNN)r&r'r(rr-r|r[r*rzr+r2rr}rr FloatTensorrrr/r/rr0rs<   rcsLeZdZUeed<dZdZdgZdZdZ dZ dZ e fddZZS)BambaPreTrainedModelr3modelTrrc sXt|t|tr*t|jt|jt t d|j dt|j dSdS)Nr)r _init_weightsrrinitones_r9rWr;r*r:rr%r=)rWrrr/r0res   "z"BambaPreTrainedModel._init_weights)r&r'r(rr,base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa _is_statefulr*rrrr/r/rr0rYs rcseZdZdeffdd Zee         ddejdBdej dBdejdBde dBd ej dBd e dBd e dBd e dBd ejdBde edefddZddZZS) BambaModelr3cst||j|_|j|_t|j|j|j|_g}t |j D]}| t |||j |dq t||_|j|_t|j|jd|_t|d|_d|_|dS)N)rarr)r3F)rr[ pad_token_id padding_idx vocab_sizer EmbeddingrP embed_tokensrLrMrTrrE ModuleListlayersrrr.final_layernormr~ rotary_embgradient_checkpointing post_init)rWr3decoder_layersrZrr/r0r[ps    zBambaModel.__init__N input_idsrrr inputs_embedsrroutput_hidden_statesrrrrfc  Ks|dur|n|jj}|dur|n|jj}|dur|n|jj}|du|duAr*td|jr9|jr9|r9tdd}|durB| |}|} |rO|durOtd| dur^t j | j d| j d} |durg| d}t|j||| ||d} ||| } |j| |d }|rd nd}|rd nd}|jD]5}|jd kr| n| }|r|| f7}|| f|||||| |d | }|d} |r|ddur||df7}q|| } |r|| f7}|r|jsd |_|sdn|}t| |||dS)Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzBamba requires an initialized `HybridMambaAttentionDynamicCache` to return a cache. None was provided, so no cache will be returned.rr9r)r3rrrrrr)rr/r4)rrrrrrrrT)last_hidden_staterr attentions)r3rrrrrrrArBrr*rrjr7rr_update_mamba_maskrrrrrFr)rWrrrrrrrrrrrr causal_mask mamba_maskrall_hidden_statesall_self_attns decoder_layer layer_mask layer_outputs next_cacher/r/r0rs               zBambaModel.forwardcCs.|}|ddks|durt|dkrd}|S)zv No need for zeroing states when 1. Cached forward 2. Attending to all inputs rNr)r*r?)rWrrrrr/r/r0rs"zBambaModel._update_mamba_mask) NNNNNNNNN)r&r'r(rr[rrr*r+rzr2rrrr rrrrr/r/rr0rnsJ    ercseZdZddiZddiZddgdgfiZfddZee dd e j d Bd e j d Bd e j d Bde d Bde j d Bde j d Bded Bded Bded Bde j d Bdee j BdefddZ  dfdd ZZS)BambaForCausalLMzlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputrlogitscsHt|t||_|j|_tj|j|jdd|_|j |_ | dS)NFr) rr[rrrrrrPrz_loss_coefficientrrrr/r0r[s   zBambaForCausalLM.__init__Nrrrrrrlabelsrrrrrlogits_to_keeprfc  Ks|dur|n|jj}| dur| n|jj} |jd |||||||| | d | } | j}t| tr4t| dn| }||dd|ddf}d}|durt|j d |||jj d| }|j dkrt|j ddj |jdd}||j |}t||| j| j| jd S) aJ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, BambaForCausalLM >>> model = BambaForCausalLM.from_pretrained("...") >>> tokenizer = AutoTokenizer.from_pretrained("...") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N) rrrrrrrrrr)rrrrrgrhrr5)lossrrrrr/)r3rrrrrr-slicer loss_functionrr logsumexprpr8rrrrrr)rWrrrrrrrrrrrrrrr slice_indicesrrz_lossr/r/r0rs@'    zBambaForCausalLM.forwardTFc sX|durt|j|jd|j|jd}|jj| d<tj|f|||||||d| } | S)Nrr9r)rrrrrrris_first_iteration)r2r3rjr8r7num_logits_to_keeprprepare_inputs_for_generation) rWrrrrrrrrrr model_inputsrr/r0rUs&   z.BambaForCausalLM.prepare_inputs_for_generation) NNNNNNNNNNr)NNNNNTF)r&r'r(_tied_weights_keys_tp_plan_pp_planr[rrr*r+rzr2rrr-rrrrr/r/rr0rsf      Pr)rrr)r)r)Kcollections.abcrtypingrrrr*rtransformers.activationsrr r cache_utilsr generationr integrationsr r integrations.hub_kernelsr masking_utilsrmodeling_layersrmodeling_outputsrrmodeling_rope_utilsrrmodeling_utilsrrprocessing_utilsrutilsrrrrr utils.genericrconfiguration_bambar get_loggerr&rAr r2Moduler~rrzr-rrrrrrr r rrrrrrrrr__all__r/r/r/r0s             kA &F  o`