o wi!@sdZddlZddlmZddlmZmZmZddlZddl Zddlm Z ddl m Z ddl mZdd lmZdd lmZdd lmZdd lmZdd lmZmZmZddlmZmZmZddlm Z e!e"Z#erqddl$m%Z%ndZ%erddl&m'Z'm(Z(ddl)m*Z*nd\Z*Z(Z'erddl+m,Z,m-Z-nd\Z-Z,e.e*e(e,e-e'fZ/Gddde j0Z1Gddde j0Z2GdddeZ3eGdddeZ4eedd Gd!d"d"eZ5eed#d Gd$d%d%eZ6eGd&d'd'e4Z7ed(d Gd)d*d*e4eZ8gd+Z9dS),zPyTorch MAMBA model.N) dataclass)AnyOptionalUnion)nn)CrossEntropyLoss)ACT2FN) MambaCache)GenerationMixin)GradientCheckpointingLayer)PreTrainedModel) ModelOutputauto_docstringlogging)is_causal_conv1d_availableis_mamba_ssm_availableis_mambapy_available) MambaConfig)pscan)mamba_inner_fnselective_scan_fn)selective_state_updateNNN)causal_conv1d_fncausal_conv1d_update)NNc seZdZdZdedeffdd Z   ddejde e d e ej d e ej fd d Z dde e d e ej d e ej fd dZ   dde e d e ej d e ej fddZZS) MambaMixeru Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`. A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective) ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4, and is why Mamba is called **selective** state spaces) config layer_idxcst||_|j|_|j|_|j|_|j|_t |j |_ ||_ |j |_ t j|j|j|j |j|j|jdd|_|j|_t|j|_|j|_t j|j|jd|jd|_t j|j|j |jddd|_t j|j |jdd|_tjd|jdtjddddf}||jd}t t ||_!t t"|j|_#t j|j|j|jd|_$|j|_t%s|jrt&rt'(d dSt)d t'(d dSdS) Nr) in_channels out_channelsbias kernel_sizegroupspaddingr"FTdtypea7The fast path is not available because one of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. Falling back to the mamba.py backend. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1dzuse_mambapy is set to True but the mambapy package is not installed. To install it follow https://github.com/alxndrTL/mamba.py.aThe fast path is not available because one of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. Falling back to the sequential implementation of Mamba, as use_mambapy is set to False. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d. For the mamba.py backend, follow https://github.com/alxndrTL/mamba.py.)*super__init__r hidden_size state_sizessm_state_size conv_kernelconv_kernel_sizeintermediate_sizeinttime_step_rankr use_conv_biasrConv1dconv1d hidden_act activationr act use_mambapyLinearuse_biasin_projx_projdt_projtorcharangefloat32expand contiguous ParameterlogA_logonesDout_projis_fast_path_availablerlogger warning_once ImportError)selfrrA __class__e/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/mamba/modeling_mamba.pyr,GsT    $zMambaMixer.__init__N hidden_states cache_paramscache_positionattention_maskcCs||dd}|jrI|durIt||jj|jr|jjnd|jj|j j|j j|j r/|j j ndt |j  dd|j |j j dd }|S|jddd\}}|dur]||d}|jj|jjd|jjd}|dur|ddkrt|d|j|j||jj|j}|d}n&|durtj||j|jddf} ||j| |t |||jj|jd}|dur||d}||dd} t j!| |j"|j#|j#gdd\} } } |j j| dd}t |j  }t$|j d r|j j nd}|dur2|ddkr2t%|j&|j|d |d || dddf| dddf|j|d |dd d}n,t'|||| dd| dd|j ||ddd \}}|dur^|dur^|(|j|| |dd}|S) Nrr&T) delta_biasdelta_softplusdimrr*)r9r").r) dt_softplus)r[return_last_state))r> transposetrainingrr7weightr5r"r?r@rKr=floatrAexprHrJchunk unsqueezeviewsizersqueeze conv_statesrr9r functionalpadr1shapeupdate_conv_statersplitr4r/hasattrr ssm_statesrupdate_ssm_state)rPrVrWrXrYprojected_statescontextualized_statesgate conv_weightsrjssm_parameters time_stepBCdiscrete_time_steprQtime_proj_bias scan_outputs ssm_staterTrTrUcuda_kernels_forwards X$          zMambaMixer.cuda_kernels_forwardc Cs|j\}}}|j}||dd} | jddd\} } |dur&| |d} |dur|j|j} | | j } |jd|j krft j | |j | jddf} ||j| |||| dd|f} nU||j| |} | |jjj } tj| |jjdddddfdd} |jr| |jj7} ||  |d} ntj||j|jf| j |d} ||| dd|f} |dur| |d} || dd}tj||j|j|jgdd\}}}||}t j |dd}t|j ! }t|ddddddf|dddddddf}|dddddddf|dddddddf!}|| dddddddf!}|j"r|j#r|durt$|dd|dd}||d%ddd}|| |j&ddddf}||| }nug}t'|D]D}|dddd|ddf| |dddd|ddf} t(| ||dd|ddfd}|)|dddddfqtj*|dd}|| |j&ddddf}||| }|dur|j|j+| |,|dd}|S) Nrr&r\rr*.devicer)r)-rmr)r>r`rerfrqrclonetorr1rrkrlrnr:r7rbrAsumr5r"zerosr2r/r?ror4r@softplusrdrHrcr;rarrirJrangematmulappendstackcopy_rK)rP input_statesrWrXrY batch_sizeseq_len_r)rsrVrur~ conv_staterwrxryrzr{rQ discrete_A discrete_BdeltaB_uhs scan_outputr}irtrTrTrU slow_forwardsp  (   :<$ <*  zMambaMixer.slow_forwardcCs>trd|jjjjvrtjs|||||S| ||||S)Ncuda) rLr?rbrtyperA_dynamo is_compilingrr)rPrVrWrXrYrTrTrUforward;szMambaMixer.forwardr)__name__ __module__ __qualname____doc__rr3r,rATensorrr LongTensorrrr __classcell__rTrTrRrUr?s4? (fUrcs.eZdZdfdd ZddZddZZS) MambaRMSNormư>cs&ttt||_||_dS)zL MambaRMSNorm is equivalent to T5LayerNorm and LlamaRMSNorm N)r+r,rrFrArIrbvariance_epsilon)rPr-epsrRrTrUr,Hs  zMambaRMSNorm.__init__cCsJ|j}|tj}|djddd}|t||j}|j||S)Nr&r*T)keepdim) r)rrArCpowmeanrsqrtrrb)rPrV input_dtypevariancerTrTrUrPs  zMambaRMSNorm.forwardcCs|jjdd|jS)Nrz, eps=)rbrmrrPrTrTrU extra_reprWszMambaRMSNorm.extra_repr)r)rrrr,rrrrTrTrRrUrGsrcsJeZdZfddZ   d deedeejdeejfddZZ S) MambaBlockcsBt||_||_|j|_t|j|jd|_t ||d|_ dS)Nrr) r+r,rrresidual_in_fp32rr-layer_norm_epsilonnormrmixer)rPrrrRrTrUr,\s zMambaBlock.__init__NrWrXrYcCsL|}||j|jjjd}|jr|tj}|j||||d}||}|S)Nr(rWrXrY)rrrbr)rrArCr)rPrVrWrXrYresidualrTrTrUrds zMambaBlock.forwardr) rrrr,rr rArrrrTrTrRrUr[s  rc@s,eZdZeZdZddgZdZdZddZ dS)MambaPreTrainedModelbackbonerrTc Cst|trd|j_d|j_|jjd|jj}|jjdkr't j |j j |n|jjdkr8t j |j j | |tt|jjt|jjt|jjt|jjj|jjd}|tt|  }t|j j|Wdn1swYd|j j_t|t jr|jdurt|jddst j |jnt|t j!rt j j"|j |jj#d |jj$r|%D]2\}}|d vrt j j&|t'd d t|t'|jj(}Wdn1swYqdSdS) zInitialize the weights.Tgconstantrandom)minN _no_reinitF)std)zout_proj.weight)a)) isinstancerrH_no_weight_decayrJrr4time_step_scaletime_step_init_schemerinit constant_r@rbuniform_rArdrandr2mathrG time_step_max time_step_minclamptime_step_floorexpm1no_gradr"rrr<getattrzeros_ Embeddingnormal_initializer_rangerescale_prenorm_residualnamed_parameterskaiming_uniform_sqrtnum_hidden_layers)rPmodule dt_init_stddtinv_dtnameprTrTrU _init_weightssN           z"MambaPreTrainedModel._init_weightsN) rrrr config_classbase_model_prefix_no_split_modulessupports_gradient_checkpointing _is_statefulrrTrTrTrUrws rz, Class for the MAMBA model outputs. ) custom_introc@sJeZdZUdZdZeejed<dZ ee ed<dZ ee ejed<dS) MambaOutputa9 cache_params (`MambaCache`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. Includes both the State space model state matrices after the selective scan, and the Convolutional states Nlast_hidden_staterWrV) rrrrrrrA FloatTensor__annotations__rWr rVtuplerTrTrTrUrs rzK Base class for causal language model (or autoregressive) outputs. c@s\eZdZUdZdZeejed<dZ eejed<dZ ee ed<dZ ee ejed<dS)MambaCausalLMOutputa loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). cache_params (`MambaCache`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. Includes both the State space model state matrices after the selective scan, and the Convolutional states NlosslogitsrWrV)rrrrrrrArrrrWr rVrrTrTrTrUrs  rcseZdZfddZddZddZddZe dd ee j d ee j d ee d ee dee dee dee j dee j de eeffddZZS) MambaModelcsnttjj|_tfddtj D|_ d|_ t jj d|_||j|dS)Ncsg|]}t|dqS)r)r).0idxrrTrU sz'MambaModel.__init__..Fr)r+r,rr vocab_sizer- embeddings ModuleListrrlayersgradient_checkpointingrrnorm_f"_register_load_state_dict_pre_hook load_hook post_initrPrrRrrUr,s    zMambaModel.__init__cGs2|D]}d|vr||||dd<dSqdS)Nz embedding.z embeddings.)popreplace)rP state_dictprefixargskrTrTrUrs zMambaModel.load_hookcC|jSNrrrTrTrUget_input_embeddingszMambaModel.get_input_embeddingscC ||_dSrrrPnew_embeddingsrTrTrUset_input_embeddings zMambaModel.set_input_embeddingsN input_ids inputs_embedsrW use_cacheoutput_hidden_states return_dictrXrYreturnc Csd|dur|n|jj}|dur|n|js|jjnd}|dur|n|jj}|du|duAr/td|dur8||}|jrB|jrB|rBd}|rk|durbt|j| d|j |j d}t j d|jj|j d}n |durjtdnd}|} |rsdnd} |jD]} | | |||d } |r| | f} qx|| } |r| | f} |std d | || fDSt| |r|| d Sd| d S) a cache_params (`MambaCache`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). use_cache (`bool`, *optional*): If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. NFz:You must specify exactly one of input_ids or inputs_embedsrrrzYou have to specify the `cache_position` manually when `use_cache=True` and `cache_params` is passed, you don't have to pass a `cache_params` if you are in prefilling stage because in that case it will be initialized for you automaticallyrTrcss|] }|dur|VqdSrrT)rvrTrTrU =sz%MambaModel.forward..)rrWrV)rrraruse_return_dict ValueErrorrrr rhrr)rArBr0rrrr) rPrrrWrrrrXrYrVall_hidden_states mixer_blockrTrTrUrs^       zMambaModel.forward)NNNNNNNN)rrrr,rrr rrrArr boolrrrrrrTrTrRrUrsB    rz The MAMBA Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). cs"eZdZdgZfddZddZddZdd Zd d Z d!d e de e e fde de e e ffddZ     d"deedeejdeejfddZe         d#deejdeejdeejdeedeejdeedeedeedeejdeeeffdd ZZS)$MambaForCausalLMzlm_head.weightcs8t|t||_tj|j|jdd|_| dS)NFr') r+r,rrrr<r-rlm_headrrrRrTrUr,Os   zMambaForCausalLM.__init__cCrrrrrTrTrUget_output_embeddingsVr z&MambaForCausalLM.get_output_embeddingscCr rrr rTrTrUset_output_embeddingsYrz&MambaForCausalLM.set_output_embeddingscCs |jSr)rrrrTrTrUr\rz%MambaForCausalLM.get_input_embeddingscCs |j|Sr)rr r rTrTrUr _s z%MambaForCausalLM.set_input_embeddingsroutputs model_kwargsnum_new_tokensrcKs|dd|d<|ddr$d|vr$|ddur$|ddd||d<d|vr?|d}tj|||jddfgdd |d<|S) NrWrTrXr*rYrrr\)getrAcatnew_onesrm)rPr"r#r$kwargsrYrTrTrU#_update_model_kwargs_for_generationbs   z4MambaForCausalLM._update_model_kwargs_for_generationNrWrXrYc Ks|r-|dur td|ddkr"|dddfd}|dur!d}n tjd|jj|jd}|dur:|dur:d|i}nd|i}|||||d|S)Nz`cache_position` should not be None as it should have been initialized in `model.generate`, you are responsible for passing in a valid `cache_position` if you are calling `prepare_inputs_for_generation` directly with `use_cache=True`rr*rrr)rWrrXrY) rrfrArBrr0rrEupdate) rPrrrrWrXrYr( model_inputsrTrTrUprepare_inputs_for_generationus,    z.MambaForCausalLM.prepare_inputs_for_generationrrlabelsrrrc  Ks|dur|n|jj}|j||||||| |d} | d} || |jjj} d}|dur]|| j}| dddddf }|dddf }t }|| d| d| d}|ss| f| dd}|durq|f|S|St || | j| jdS)aS cache_params (`MambaCache`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` use_cache (`bool`, *optional*): If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits. N)rWrrrrrXrYr.r*r)rrrWrV)rrrrrrbr)rcrrErrgrhrrWrV)rPrrYrrWr-rrrrXr( mamba_outputsrVrr shift_logits shift_labelsloss_fctoutputrTrTrUrs:  zMambaForCausalLM.forward)r)NNNNN) NNNNNNNNN)rrr_tied_weights_keysr,r r!rr rdictstrrr3r)rr rArr,rrrrrrrrrrTrTrRrUrFsx     0   r)rrr):rr dataclassesrtypingrrrrAtorch.utils.checkpointrtorch.nnr activationsr cache_utilsr generationr modeling_layersr modeling_utilsr utilsrrrutils.import_utilsrrrconfiguration_mambar get_loggerrrM mambapy.pscanr&mamba_ssm.ops.selective_scan_interfacerr+mamba_ssm.ops.triton.selective_state_updater causal_conv1drrallrLModulerrrrrrrr__all__rTrTrTrUsn             7i