o پim.@sdZddlZddlmZddlmZmZmZmZm Z m Z m Z ddl Z ddlZddlmmZddlmmmZddlZddlmZddlmZddlmZddlmZmZmZmZdd l m!Z!dd l"m#Z#m$Z$dd l%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+dd l,m-Z-ddl.m/Z/m0Z0ddl1m2Z2m3Z3m4Z4ddl5m6Z6ddl7m8Z8ddl9m:Z:ddl;mZ>m?Z?ddl@mAZAddlBmCZCzddlmDZDddlEmFZFdZGWndZDdZGY     dGdejHdejHdejHdeId eIf d!d"ZJeGd#d$d$e'ZK % & '  dHd(ejHd)eId*ejHd+eId,eId-eId.eId/eLd0ejHfd1d2ZMGd3d4d4ejNZOGd5d6d6ejNZPGd7d8d8ejNZQGd9d:d:ejNZRGd;d<d<ZSGd=d>d>eZTGd?d@d@ejNZUGdAdBdBe+ZVGdCdDdDejNZWGdEdFdFe>ZXeXgZYdS)IzCInference-only MiniCPM-o model compatible with HuggingFace weights.N) dataclass)AnyIterableListLiteralOptionalTupleUnion)nn)parametrizations)tqdm) LlamaConfig LlamaModelPretrainedConfigPreTrainedModel)ACT2FN) DynamicCacheEncoderDecoderCache)BaseModelOutputWithPast ModelOutput)WhisperAttention WhisperConfigWhisperEncoder)QuantizationConfig))MultiModalityDataPaddingPatternTokenPairsgeneral_mm_embed_routine)MultimodalDataItemMultimodalInputsflatten_nested_list) ForwardBatch)set_default_torch_dtype)default_weight_loader)Idefics2VisionTransformer)MiniCPMBaseModel Resampler2_5Qwen2ForCausalLM)logger) LogitsWarper)GroupedResidualFSQTF input_idsspk_emb input_embedsspk_emb_token_id num_spk_embsc Cs||jd}t|D]2}||}||}||k} | jdd} | jd|ks&J| } | } |||| | dddf<q dS)a Replace consecutive `num_spk_embs` speaker embedding placeholders in input_embeds with pre-prepared speaker embeddings. This is an in-place replacement, no new tensor is created, so no value is returned. Args: input_ids (torch.Tensor): Input ID tensor, shape [batch_size, seq_len_max] spk_emb (torch.Tensor): Speaker embedding tensor, shape [batch_size, num_spk_emb, hidden_dim] input_embeds (torch.Tensor): Input embedding tensor, shape [batch_size, seq_len_max, hidden_dim] spk_emb_token_id (int): ID of the speaker embedding token num_spk_embs (int): Number of speaker embeddings Returns: None rF)as_tupler*N)shaperangenonzerominmax) r+r,r-r.r/ batch_sizeidx input_ids_spk_emb_mask_nonzero_position_idx begin_idxend_idxr>N/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/models/minicpmo.py apply_spk_embDs   r@c@sTeZdZUdZdZejed<dZejed<dZ e e e ej ed<dZ eed<dS)"ConditionalChatTTSGenerationOutputa Output class for ConditionalChatTTS generation. Args: new_ids (torch.LongTensor): Newly generated audio code sequence, shape (batch_size, sequence_length, num_vq). audio_input_ids (torch.LongTensor): Updated input IDs including condition and generated audio codes, shape (batch_size, full_sequence_length, num_vq). past_key_values (Tuple[Tuple[torch.FloatTensor]]): Tuple containing pre-computed keys and values used for attention mechanism. Each element has shape (batch_size, num_heads, sequence_length, embed_size_per_head). finished (bool): Boolean indicating whether generation is complete. Nnew_idsaudio_input_idspast_key_valuesfinished)__name__ __module__ __qualname____doc__rBtorch LongTensor__annotations__rCrDrr FloatTensorrEboolr>r>r>r?rAhs  rA,2 inputs_embedspast_seen_tokensstreaming_tts_text_maskstreaming_reserved_lengthstreaming_audio_chunk_sizestreaming_text_chunk_size num_spk_emb use_spk_embreturncCs|jddks J|j}|j} t|j} tjd||jdfd|| d} tt|||||d||} |d||d} | | d| | f<| ddd|||df |dk| | d d} | S)a In streaming audio generation, determine which `text` positions the TTS model can attend to when generating each chunk of `audio` tokens. This function creates a mask that allows the model to attend to a specific chunk of text tokens when generating each chunk of audio tokens, enabling streaming TTS generation. Args: inputs_embeds (torch.Tensor): Input embeddings tensor. past_seen_tokens (int): Number of tokens already seen by the model. streaming_tts_text_mask (torch.Tensor): Mask for the text tokens. streaming_reserved_length (int, optional): Number of reserved tokens for streaming. Defaults to 300. streaming_text_chunk_size (int, optional): Size of each text chunk. Defaults to 7. Returns: torch.Tensor: Causal mask for streaming TTS generation, shape is [batch_size=1, 1, seq_len=1, past_seen_tokens+1] Raises: AssertionError: If the batch size is not 1 (only supports batch size of 1 for inference). rr*) fill_valuedtypedevice) r1r\r]rJfinfor4fullmathceil masked_fill_ unsqueeze)rRrSrTrUrVrWrXrYr\r] min_dtype causal_maskinvisible_text_tokens_startinvisible_text_tokens_endr>r>r?$make_streaming_chunk_mask_generation{sD   rhc sNeZdZ ddededededef fdd Zdd ejd ejfd d ZZ S) ConvNeXtBlockư>dimintermediate_dimkerneldilationlayer_scale_init_valuecsttj|||||d||d|_tj|dd|_t|||_t |_ t|||_ |dkrCtj |t |dd|_dSd|_dS)N) kernel_sizepaddingrngroupsrj)epsrT) requires_grad)super__init__r Conv1ddwconv LayerNormnormLinearpwconv1GELUactpwconv2 ParameterrJonescoef)selfrkrlrmrnro __class__r>r?rws$    zConvNeXtBlock.__init__NxrZcCsx|}||}|dd||}~||}~||}~||}~|jdur/||j9}|dd||}~|S)Nr*rp)ry transpose_r{r}rrr)rrcondresidualyr>r>r?forwards"         zConvNeXtBlock.forward)rjN) rFrGrHintfloatrwrJTensorr __classcell__r>r>rr?ris ricsLeZdZ      ddedeffd d Zdd ejd ejfddZZS) DVAEDecoder @rpFidimodimc szt||_tt||dddtt|ddd|_tfddt |D|_ tj|ddd|_ dS)Nr*csg|] }tdqS))ri.0_rnhiddenrmr>r? sz(DVAEDecoder.__init__..F)rqbias) rvrwupr Sequentialrxr~conv_in ModuleListr2 decoder_blockconv_out) rrrn_layerbn_dimrrmrnrrrr?rws  zDVAEDecoder.__init__NrrZcCs2||}~|jD]}|||}q ||}~|Sr)rrr)rr conditioningrfr>r>r?rs    zDVAEDecoder.forward)rrrrrpFr) rFrGrHrrwrJrrrr>r>rr?rs rc sxeZdZ  ddedeededeffdd Zd ejfd d Zd ejd ejffd d Z d ejd ejfddZ Z S)GFSQh㈵>TrklevelsGRcsLtt|t|t|||d|_t||_||_ ||_ ||_ ||_ dS)N)rkrnum_quantizersrs) rvrrwr)list quantizerr`prodn_indrt transposerr)rrkrrrrtrrr>r?rw+s   z GFSQ.__init__rcCs`|jr |dd}||d|d|j|jdddd}|j|}|jr.|ddS|S)Nr*rprr) rviewsizerrpermuterget_output_from_indicesr)rrfeatr>r>r?_embedAs  , z GFSQ._embedrZcs t|Sr)rv__call__)rrrr>r?rHs z GFSQ.__call__cCsd|jr |dd||\}}|dddd}||d|dd}|jr0|ddS|S)Nr*rprr)rrrr contiguousrr)rrrindr>r>r?rKs  z GFSQ.forward)rT) rFrGrHrrrwrJrrrrrr>r>rr?r*srcsFeZdZfddZe d dejdeddejfdd ZZ S) DVAEc sttd}t|dd|_t t dddddt t dddddt |_ t ddd d d d |_t ddd d d d |_tj dddddd d|_tddddd|_dS)Ndrrpirr*rirr)rrrrrFr)rrr)rkrrr)rvrwrJrandr rrc unsqueeze_rrrxr~downsample_convrencoderdecoderout_convrvq_layer)rrrr>r?rwVs<   z DVAE.__init__decodeinpmode)encoderrZcCs|dkr7t|dr7|jdur7|}|tj||jdd|j |d d}~| |}||}~|S|jdurC|j |}n|}|| dd| dd| dfddddd}||j|d }~tj||j|dS) Nrrrr*)outrrpr)r)hasattrrclonerrJdivrrexpandr1rrrrrflattenrrmul)rrrmelrrvq_featsdec_outr>r>r?r~s8      z DVAE.forward)r) rFrGrHrwrJinference_moderrrrr>r>rr?rUs (rc@s>eZdZdededefddZdejdejdejfd d Z d S) ,CustomRepetitionPenaltyLogitsProcessorRepeatpenalty max_input_ids past_windowcCs6t|tr |dkstd|||_||_||_dS)Nrz6`penalty` has to be a strictly positive float, but is ) isinstancer ValueErrorrrr)rrrrr>r>r?rws z5CustomRepetitionPenaltyLogitsProcessorRepeat.__init__r+scoresrZc Cs|d|jkr|d|j |j}t||dd}|d|jkr6|d|j|d|jt |j |}| }| |}| |}|dk}t|||}~~~~~|S)Nr*r)rrnarrowFone_hotsumrzero_rJpowrrmultiplydividewhere) rr+rfreqalpharothconrr>r>r?rs    z5CustomRepetitionPenaltyLogitsProcessorRepeat.__call__N) rFrGrHrrrwrJrKrMrr>r>r>r?rs rcsNeZdZdZeZgZdeffdd Ze ddej de ej fdd Z e ddej d ej d eeej ej fde ej fd d Ze  ddej d eeej ej fdefddZe       d dej d eeej ej fdej deeej fde eede eef ddZe deej fddZZS)!ConditionalChatTTSadA conditional text-to-speech model that can generate speech from text with speaker conditioning. This model extends PreTrainedModel to provide text-to-speech capabilities with: - LLM hidden state conditioning - Streaming generation The model uses a transformer architecture with LLM hidden states and can operate in both streaming and non-streaming modes for flexible deployment. The model process sequence in the following format: | text bos token | LLM embedding projected to tts embedding space | text tokens (fixed length, reserved for future tokens) | audio bos token | audio tokens (audio token length is not fixed)| audio eos token | The format is designed to support LLM-conditioned streaming audio generation. Usage: To support streaming generation, two global variables should be maintained outside of the model. 1. `audio_input_ids`: stores *discrete* audio codes. It is a tensor with shape [1, sequence length+1, num_vq]. 2. `past_key_values`: stores the KV cache for both text tokens and audio codes. It is a list of tuples, each tuple contains two tensors with shape [1, num_attention_heads, sequence length, hidden_size // num_attention_heads] where `num_vq` is the number of audio codebooks, in default setting, it is `4`. 1. Create an empty `past_key_values` with ```python initial_kv_cache_length = 1 + model.num_spk_embs + model.streaming_text_reserved_len # where `1` denotes the `bos` token dtype = model.emb_text.weight.dtype device = model.emb_text.weight.device past_key_values = [ ( torch.zeros(1, model.config.num_attention_heads, initial_kv_cache_length, model.config.hidden_size // model.config.num_attention_heads, dtype=dtype, device=device), torch.zeros(1, model.config.num_attention_heads, initial_kv_cache_length, model.config.hidden_size // model.config.num_attention_heads, dtype=dtype, device=device) ) for _ in range(model.config.num_hidden_layers) ] 2. At the same time, create an empty `audio_input_ids` with shape [1, sequence length, num_vq], `num_vq` denotes multiple layer audio codebooks. But here we also include text tokens in the sequence, but they will be zeros, and will not be used, just a placeholder. ```python initial_audio_input_ids_length = 1 + model.num_spk_embs + model.streaming_text_reserved_len + 1 # [bos token, speaker embeddings, text tokens, audio bos token] audio_input_ids = torch.zeros(batch_size=1, initial_audio_input_ids_length, model.num_vq) ``` 2. Prefill some text tokens to TTS model (for example, 10 tokens) using `prefill_text` method. ```python outputs = llm.generate(**kwargs) llm_tokens = some_function_to_extract_llm_tokens(outputs) lm_spk_emb_last_hidden_states = some_function_to_extract_lm_spk_emb_last_hidden_states(outputs) tts_text_input_ids = tts_tokenizer.encode(llm_tokenizer.decode(llm_tokens)) # here assume we are prefilling text token 0 to text token 9 (included), totally 10 tokens. begin = 0 end = 9+1 position_ids = torch.arange(begin, end, dtype=torch.long, device=device) past_key_values = model.prefill_text( input_ids=tts_text_input_ids, position_ids=position_ids, past_key_values=past_key_values, lm_spk_emb_last_hidden_states=lm_spk_emb_last_hidden_states, ) ``` 3. Make a `streaming_tts_text_mask` to denote which position contains valid text tokens, similar to `attention_mask` in standard causal attention. ```python streaming_tts_text_mask = torch.zeros(model.streaming_reserved_length) streaming_tts_text_mask[0:end] = 1 # denotes these post ``` 3. Generate audio codes using `generate` method. ```python outputs = model.generate( input_ids=audio_input_ids, past_key_values=past_key_values, streaming_tts_text_mask=streaming_tts_text_mask, max_new_token=50, ) # update past_key_values and input_ids past_key_values = outputs.past_key_values audio_input_ids = outputs.input_ids ``` The `past_key_values` is extended by `max_new_token=50`, and `audio_input_ids` is also extended by `max_new_token=50` after `generate` calling. 4. Notice that after prefilling `10` text tokens, the model can generate up to `50` audio tokens, if you want to generate more audio tokens, you need to prefill next `10` text tokens. And it is okay to only generate `25` audio tokens for faster initial response. 5. Repeat steps `2,3,4` as needed in your streaming audio generation cases, but ensure usage complies with the following guidelines discussed above. configcsLtj|_j|_j|_j|_j|_j|_j|_j |_ j |_ j |_ j |_ j |_ j|_j|_j|_j|_|jjrStjj|_n tjjjdd|_tfddtj D|_tjj|_tfddtj D|_ t!}||_"t#jj$j%j&j'j(d}t)|}||_*dS)NFrcsg|] }tjjqSr>)r Embeddingnum_audio_tokens hidden_sizerrr>r?r?sz/ConditionalChatTTS.__init__..cs*g|]}tjtjjjddddqS)Frweight)name)r weight_normr r|rrrrr>r?rFs)rintermediate_sizenum_attention_headsnum_hidden_layersmax_position_embeddingsattn_implementation)+rvrwuse_speaker_embeddinguse_llm_hidden_stater/r.use_text streamingrWrVstreaming_text_reserved_lenaudio_bos_token_id num_mel_binsnum_vqrtop_ptop_krepetition_penaltyruse_mlpMultiModalProjectorllm_dimr projectorr r|rr2emb_codernum_text_tokensemb_text head_coderdvaer rrrrrrmodel)rrr model_configrrrr?rw$sV     zConditionalChatTTS.__init__Nr+lm_spk_emb_last_hidden_statescCs|jddks J||}|jrC||jk}|rA|dus J||jjjj }||}t j |ddd}t ||||j|j d|St)aMerge `input_ids` and `lm_spk_emb_last_hidden_states` to `inputs_embeds`. Args: input_ids (torch.Tensor): Input token IDs. lm_spk_emb_last_hidden_states (Optional[torch.Tensor], optional): Last hidden states of speaker embeddings from the language model. Defaults to None. Raises: NotImplementedError: If speaker embedding is not used and language model hidden states are not implemented. Returns: torch.Tensor: Prepared input embeddings for the model. rr*Nrpr)prk)r+r,r-r.r/)r1rrr.anytor linear1rr\r normalizer@r/NotImplementedError)rr+rrR spk_emb_maskprojected_spk_embr>r>r?merge_inputs_embeds^s(      z&ConditionalChatTTS.merge_inputs_embeds position_idsrDc Cs|jddks J|dusJ|j||d}g}tt|D]=}|||dddddd|dddfddf||dddddd|dddfddffq|jd|||dd|d}|j} tt|D]} | | ddddd|dddf|dddfdf|| ddddd|dddf|dddfdddf<| | ddddd|dddf|dddfdf|| ddddd|dddf|dddfdddf<qq|S) aPrefill a chunk of new text tokens in streaming setting. Specifically speaking, update `past_key_values` using new text tokens, then the model will read the new text tokens. Args: input_ids (Tensor): Tensor of shape [batch_size, seq_len] position_ids (LongTensor): Tensor of shape [batch_size, seq_len] past_key_values (List[Tuple[Tensor]]): KV Cache of all layers, each layer is a tuple (Tensor, Tensor) denoting keys and values. Each tensor is of seq_len = `self.streaming_text_reserved_len`. `past_key_values` will be updated. lm_spk_emb_last_hidden_states (Tensor, optional): Tensor of shape [batch_size, num_spk_emb, llm_dim]. Defaults to None. Note that all `batch_size` should be `1`. rr*N)r+rTFattention_maskrrDrR use_cacheoutput_attentionscache_positionr)r1rr2lenappendrrrD) rr+rrDrrRpast_key_values_for_prefillioutputs_prefill#past_key_values_for_prefill_updated layer_idxr>r>r? prefill_textsN 66  0 6 0 6 zConditionalChatTTS.prefill_textT add_audio_bosc sjddks J|dusJfddtjD}t|dd}jd}|rKtjjggtjj d} |} tj | |gdd}|d7}|ddjd } tj | | |tjj d d} | } t||ddjd |jjd } j| | ||d d | d }|j}|S)aPrefill a chunk of audio ids to the model. Used in sliding-window long audio generation. Specifically, prefill many audio ids (typically from last window) to the model in the new window. Args: input_ids (torch.Tensor): (1, seq_len, num_vq) Audio input token ids. past_key_values (List[Tuple[torch.Tensor, torch.Tensor]]): Past key values for attention mechanism. rr*Nc,g|]}j|dddd|fqSrr rr'r+rr>r?rs,z8ConditionalChatTTS.prefill_audio_ids..rr\r]rkrprRrSrTrUrWTFr)r1r2rrJstackrtensorrlongr]rcatarangercrrhrrWrrD)rr+rDrTr,code_embrR input_lennarrowed_input_idsbos_inputs_embedspast_key_values_lengthrr#reoutputsr>r0r?prefill_audio_idssP     z$ConditionalChatTTS.prefill_audio_idsFrQrP temperature eos_tokenlogits_warperslogits_processorsc & s|jddks J|dusJ| pg} | pg} djjjd} tj|jd|jd} |d |jdd dd}|jd}tj|jd|||jd|j |jd}| dd||~| dd|}d}| ryt|dd d }djjjd}t|D]}d }||krd }||ddjddksJ|rtjjggtjjd}n#|j d|jdddd fddtjD}t|dd}tj|ddjdgtjjdd}|}t||ddjd|jjd}j||||d d |d}~~~~|j}|j}t4tj |!d|!dj"jtj#jd}tjD]}j$||}||d|f<~q=Wdn 1sZwY~| ddd%d#}|&ddd}|'d|!d}| d| |!d| &ddd}|'|!d|!dd(j}~||}|s| D]} | ||}q|s| D]}!|!||}q~||krtj) |dd|f<|rtj) |dd|f<t*j+|dd}"~tj,|"dd}#~"|# dj}#|#-|.d}$| /|$~$| d|d|#0d|dkr| .rn~#|d7}| dd|}| 1r.n |dur8|2dq|durB|3| 1sR| rRt45d|~| 1rf|dd|dddf}%n |dd|dddf}%t6|%||| 1dS)aGenerate audio codes in streaming setting or non-streaming setting. Specifically speaking, generate audio codes when not all text tokens are prefilled. Always pass a valid `past_key_values` to the method. The method does not do `prefill` by itself. It relies on `prefill_text` method to provide valid `past_key_values`. Please refer to docstring of this class for more details. In this method, we borrowed a lot of codes from `https://github.com/2noise/ChatTTS/blob/main/ChatTTS/model/gpt.py`. Args: input_ids (torch.Tensor): Input token ids. past_key_values (List[Tuple[torch.Tensor, torch.Tensor]]): Past key values for attention mechanism. temperature (torch.Tensor): Temperature for sampling. eos_token (Union[int, torch.Tensor]): End of sequence token. streaming_tts_text_mask (Optional[torch.Tensor], optional): Mask for streaming TTS text. Defaults to None. max_new_token (int, optional): Maximum number of new tokens to generate. Defaults to 50. logits_warpers (List[LogitsWarper], optional): List of logits warpers. Defaults to []. logits_processors (List[CustomRepetitionPenaltyLogitsProcessorRepeat], optional): List of logits processors. Defaults to []. show_tqdm (bool, optional): Whether to show progress bar. Defaults to True. Returns: GenerationOutputs: Generation outputs. rr*Nr]rrpr1codezG{l_bar}{bar}| {n_fmt}/{total_fmt}(max) [{elapsed}, {rate_fmt}{postfix}])totaldesc bar_formatFT)rkstartlengthcr-rr.r/r;rr>r?rsz/ConditionalChatTTS.generate..rr3r.r2) num_samplesz&incomplete result. hit max_new_token: )rBrCrDrE)7r1r/rrrJzerosr]rNrcrrrr\rcopy_r r2r5rr6rrr4rrrhrWrlast_hidden_staterDPcachedemptyrrrrsqueeze_rreshaperinfrsoftmax multinomialeqr logical_or_rallupdatecloser'inforA)&rr+rDr@rArT force_no_stop min_new_token max_new_tokenrBrC show_tqdm start_idxfinishprogress input_ids_bufpbarcondition_lengthr' audio_bosrRr9rr#rer> hidden_stateslogits num_vq_iterrinput_ids_sliced logits_tokenlogitsProcessors logitsWarpersridx_next finish_orgenrated_input_idsr>rKr?generatesZ(                     zConditionalChatTTS.generate result_listc Cs|j}d}t|dkrtjgtjdS|D]}|d|kr#|d}qtjt||dd|f|dj|dj d}t t|D]}||}|| dd|d | dd~qB||}~|S)a-Decode discrete audio codes to mel spectrograms. Borrowed from `https://github.com/2noise/ChatTTS/blob/main/ChatTTS/core.py` Args: result_list (List[torch.Tensor]): Audio codes output from `generate`. Returns: torch.Tensor: Mel spectrograms. rrr\r*r1)rr$nparrayfloat32rrJrMr\r]r2rrNr) rrtr max_x_lenresult batch_resultr'src mel_specsr>r>r?decode_to_mel_specss(  &z&ConditionalChatTTS.decode_to_mel_specsr)NT)NFrQrPNNF)rFrGrHrIr config_class_no_split_modulesrwrJrrrrrKrrr+rNr?r rr(rrsr~rr>r>rr?rs|[: - G ;     rcsfeZdZddedeffdd Z   ddejdejd ejd ed e e d e ed ejfddZ Z S)MiniCPMWhisperEncoderLayerNrr*cst|j|_t|j|j|j||d|_t |j|_ |j |_ t |j |_|j|_t|j|j|_t|j|j|_t |j|_dS)N) embed_dim num_headsdropoutrr*)rvrwd_modelrrencoder_attention_headsattention_dropout self_attnr rzself_attn_layer_normrractivation_function activation_fnactivation_dropoutr|encoder_ffn_dimfc1fc2final_layer_norm)rrr*rr>r?rwDs  z#MiniCPMWhisperEncoderLayer.__init__Frir layer_head_maskr"rDr!rZc Cs |}||}|j|||||d\}}tjj||jdd}||}|}||}|||}tjj||jdd}| |}tjj||jdd}||}|j t j krst |sat |rst |j jd} t j|| | d}|f} |r}| |f7} |r| |f7} | S)a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, embed_dim)`): Hidden states to be fed into the encoder layer. attention_mask (`torch.FloatTensor` of shape `(batch_size, 1, tgt_len, src_len)`): Attention mask where padding elements are indicated by large negative values. layer_head_mask (`torch.FloatTensor` of shape `(encoder_attention_heads,)`): Mask to nullify selected heads of the attention modules. output_attentions (`bool`, *optional*): Whether or not to return the attention weights. past_key_values (`EncoderDecoderCache`, *optional*): Past key-value pairs used for incremental decoding. use_cache (`bool`, *optional*): Whether or not to return updated `past_key_values` for caching. Returns: A tuple of shape `(hidden_states, optional(attn_weights), optional(past_key_values))`. )rir rr"past_key_valueFrtrainingi)r4r5)rrr functionalrrrrrrr\rJfloat16isinfrisnanr^r5clamp) rrir rr"rDr!r attn_weights clamp_valuer>r>r>r?rVsN         z"MiniCPMWhisperEncoderLayer.forwardr)FNF) rFrGrHrrrwrJrrNrrrrr>r>rr?rCs(rcsLeZdZdeffdd Z       d deedeefddZZ S) MiniCPMWhisperEncoderrcs0ttfddtjD|_dS)Ncsg|]}t|dqS))r*)rr/rr>r?rs z2MiniCPMWhisperEncoder.__init__..)rvrwr rr2encoder_layerslayers)rrrrr?rws   zMiniCPMWhisperEncoder.__init__NrDr!c  Cs|dur|n|jj}|dur|n|jj}|dur|n|jj}|j|jjj|jjjd}t j ||} t j | | } | ddd} |jj} d} |r|durXttt}nt|trgtt|t}n t|trst|t}n |j| jd} | jd| | jdkrtd| | dddf} t| tj| dddfd| jd| jd| ddf} n| | | jd| ddf} n | d| jdddf} | | } t j j| |jd d } |rd nd}|rd nd}|dur|dt|j ksJd t|j d |ddt!|j D]E\}}|r$|| f}d }|r,d}n|| ||dur8||nd|||d}|d} |rP||rLdnd}nd}|r\||df}q|"| } |rk|| f}|szt#dd| ||fDSt$| |||dS)a Forward pass of the Whisper encoder. Args: input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`): Float values of log-mel features extracted from the raw audio waveform. Typically generated by a feature extractor (e.g., `WhisperFeatureExtractor`) that processes `.flac` or `.wav` files into padded 2D mel spectrogram frames. These features are projected via convolution layers (`conv1` and `conv2`) and then transformed into embeddings for the encoder. attention_mask (`torch.Tensor`, *optional*): Not used by Whisper for masking `input_features`, but included for API compatibility with other models. If provided, it is simply ignored within the model. By default, Whisper effectively ignores silence in the input log-mel spectrogram. head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected attention heads. The elements should be either 1 or 0, where: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked** (i.e., the attention head is dropped). output_attentions (`bool`, *optional*): Whether or not to return the attention tensors of all encoder layers. If set to `True`, the returned tuple (or `BaseModelOutputWithPast`) will contain an additional element with attention weights for each encoder layer. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. If set to `True`, the returned tuple (or `BaseModelOutputWithPast`) will contain a tuple of hidden states, including the initial embedding output as well as the outputs of each layer. return_dict (`bool`, *optional*): Whether or not to return a `BaseModelOutputWithPast` (a subclass of `ModelOutput`) instead of a plain tuple. If set to `True`, the output will be a `BaseModelOutputWithPast` object, otherwise it will be a tuple. past_key_values (`EncoderDecoderCache`, *optional*): When using caching for faster inference, this is an object that stores the key-value pairs for attention states. If provided, the model will append new states to the existing cache and return the updated cache. This speeds up sequential decoding or chunked inference. - If `past_key_values` is `None`, no past states are used or returned. - If `past_key_values` is not `None` and `use_cache=True`, the model will use the provided cache and return the updated cache (as `next_encoder_cache`). use_cache (`bool`, *optional*): Whether or not the model should use caching (`past_key_values`) to speed up processing during inference. When set to `True`, the model will: - Inspect and use `past_key_values` if provided. - Return updated `past_key_values` (under the name `next_encoder_cache` in `BaseModelOutputWithPast`). Returns: `BaseModelOutputWithPast` or `tuple` (depending on `return_dict`): If `return_dict=True`, a `BaseModelOutputWithPast` is returned, which contains: - **last_hidden_state** (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): The output of the final encoder layer. - **hidden_states** (`tuple(torch.FloatTensor)`, *optional*, returned if `output_hidden_states=True`): Hidden states of the model at each layer (including the initial projection). - **attentions** (`tuple(torch.FloatTensor)`, *optional*, returned if `output_attentions=True`): Attention weights from each encoder layer. - **past_key_values** (an object of type `EncoderDecoderCache` or `None`, *optional*): Updated cache of key-value pairs if `use_cache=True`. If `return_dict=False`, a tuple is returned, where the format is: `(last_hidden_state, hidden_states, attentions)`, with `hidden_states` and `attentions` only present if their respective `output_*` arguments are set to `True`. Nr1rrpr*zHseems the audio is longer than 30s. repeating the last part of the audiorr2Frr>z&The head_mask should be specified for z layers, but it is for .)NN)rr"rDr!css|] }|dur|VqdSrr>)rvr>r>r? psz0MiniCPMWhisperEncoder.forward..)rOri attentionsrD)%rr"output_hidden_statesuse_return_dictrconv1rr\r]r rgeluconv2rembed_positionsrrrrfrom_legacy_cacheself_attention_cacheget_usable_lengthr1r'warningrJr7repeat_interleavercrrr$r enumerate layer_normtupler)rinput_featuresr  head_maskr"r return_dictrDr!rR embed_posr=embed_pos_frontriencoder_statesall_attentionsr7 encoder_layerto_drop layer_outputsnext_encoder_cacher>r>r?rsQ            zMiniCPMWhisperEncoder.forward)NNNNNNN) rFrGrHrrwrrrNrrr>r>rr?rs  rcs$eZdZfddZddZZS)r cs<ttj||dd|_t|_tj||dd|_dS)NT) in_features out_featuresr)rvrwr r|rReLUrelulinear2)rin_dimout_dimrr>r?rw~s  zMultiModalProjector.__init__cCs|||}||}|Sr)rrr)raudio_featuresrir>r>r?rs zMultiModalProjector.forward)rFrGrHrwrrr>r>rr?r }s r c seZdZ d:dedeeddffdd ZddZd d Z  d;dedeed e de j fd dZ ddeefd.d/Zdeedejfd0d1Zdejd2ejd3ed4e dejf d5d6Z!d7e"e#e ejffd8d9Z$Z%S)?MiniCPMONr quant_configrZcstj||d|j||d|_|jjj|_|jjr0|j||d|_ |j j|_ | |j|j |_ d|j_ |jj r]||_t|jjjd}tj|jj|jjd|_t||jd|_d|_d|j_td|jjrwtspJd ||_dSdS) N)rrTr)stride)rrrFzTTS is disabled for nowzAplease make sure vector_quantize_pytorch and vocos are installed.)rvrwinit_llmllmrrr init_visioninit_vision_modulevpm vision_diminit_resampler resampler init_audioinit_audio_moduleapmrrr AvgPool1daudio_pool_stepaudio_avg_poolerr audio_projection_layeraudio_encoder_layerinit_ttsr'r] _tts_depsinit_tts_moduletts)rrraudio_output_dimrr>r?rws6     zMiniCPMO.__init__cCt|jj}|Sr)rr tts_configrrr>r>r?r zMiniCPMO.init_tts_modulecCrr)rr audio_configrr>r>r?rrzMiniCPMO.init_audio_moduleprefixcCst|||dS)Nrrrr%)rrrrr>r>r?rszMiniCPMO.init_llmcCsr|jjdkr d|jj_nd|jj_t|j||d}|jjr'|jjdd|j_t|d|jj t|d|jj |S)Nflash_attention_2eagerrrr patch_size) r_attn_implementation vision_configr"drop_vision_last_layerrrsetattr embeddingsrr)rrrrrr>r>r?rs   zMiniCPMO.init_vision_modulerrc CsXttjt|jj||d|||d}Wdn1swY|jdtdS)Nr) num_queriesrrkv_dimrrcudar]r\)r rJrr$r query_numrget_default_dtype)rrrrrrr>r>r?rs  zMiniCPMO.init_resamplerr+mm_inputc CsT|j}|j}|j}|j}||f||f|j|jfg}||jg}t||d} | ||S)N)data_token_pairsdata_start_token_ids) im_start_id im_end_idslice_start_id slice_end_idaudio_start_id audio_end_idrpad_input_tokens) rr+rrrrrrrpatternr>r>r? pad_input_idss   zMiniCPMO.pad_input_ids input_lengthscCs>|ddd}||jj|jjd}|jtjd}||fS)zs Computes the output length of the convolutional layers and the output length of the audio encoder r*rpru)rrrrJint32)rrinput_lengths_after_cnninput_lengths_after_poolingr>r>r? _get_feat_extract_output_lengthss z)MiniCPMO._get_feat_extract_output_lengthsitemsc Cstdd|D}tdd|D}t|dkrt|}|j\}}}|dks)J|ddd}|jdur_|jddjd} |jjjjd} | || kr_t d| |d | d d|_|j||jd d } | j } | j |_| | } | dd} || } | dd} ||\}}|}g}d}tt|D](}g}tt||D]}|| |d||ddf|d7}q||q|SgS) a Extract audio embeddings in a streaming manner using cached key-value pairs. This method processes incoming audio features incrementally and stores/updates `past_key_values` for faster inference on subsequent audio frames. It only supports batch_size=1 and is intended for streaming scenarios. Returns: List[List[torch.Tensor]]: audio embeddings cSg|]}|jr|jqSr>featureritemr>r>r?rz:MiniCPMO.get_audio_embedding_streaming..cSrr>audio_feature_lensrr>r>r?rr rr*rpNzaudio_past_key_values length z exceed z, reset.T)rDr!)rr$rJhstackr1audio_past_key_valuesrrrr'rrOrDrrrrr2r%)rrwavformsaudio_feature_lens_rawr r6rmax_mel_seq_len max_seq_len cache_length apm_max_len audio_outputs audio_states audio_embedsfeature_lens_after_poolingrfinal_audio_embedsr7r'target_audio_embedsr>r>r?get_audio_embedding_streaming sV              z&MiniCPMO.get_audio_embedding_streamingrcpurr chunk_sizenum_left_chunksr] num_lookheadc Csttj|||tjd}t|D])}|dkrd}n t||||d}t||d|||} d|||| f<q|S)aCreate mask for subsequent steps (size, size) with chunk size, this is for streaming encoder Args: size (int): size of mask chunk_size (int): size of chunk num_left_chunks (int): number of left chunks <0: use full chunk >=0: use num_left_chunks device (torch.device): "cpu" or "cuda" or torch.Tensor.device Returns: torch.Tensor: mask rrr*T)rJrMrNr2r5r4) rrrrr]rretr'rIendingr>r>r?subsequent_chunk_maskPs zMiniCPMO.subsequent_chunk_maskc Cstdd|D}tdd|D}|rIt|dtjr$dd|D}n%t|dtrIg}|D]}t|tr<||q/||q/dd|D}g}t|tsRJt|dtjs\J|D]}t|dkrNg} |D]}t|trx| |qk| |qkt| } |j \} } } | ddd}tj d|| j | j d  d| |}| d| |}||k}|| dd|| d||}|j|jjjj |jjjj d }|dkrt|d }|j||d |j d }t|t|}td ||<|j|d|dj|j}||}|dd}||}|dd}|| \} }|}d}tt|D]*}g}tt||D]} |||d||ddf|d7}q/||q#|SdS)as Extract full audio embeddings with optional chunk-based attention. This method computes embeddings for all audio frames at once, either using full attention (when `chunk_length` is -1) or chunk-based attention (when `chunk_length` is a positive number). It does not use key-value caching and is suitable for non-streaming inference. Args: chunk_length (int, optional): Determines whether to use full attention (-1) or chunk-based attention (>0) during embedding computation. Returns: List[List[torch.Tensor]]: audio embeddings cSrr>rrr>r>r?rr z0MiniCPMO.get_audio_embedding..cSrr>r rr>r>r?rr rcSsg|]}|gqSr>r>)rlensr>r>r?rcSs g|] }t|ts |gn|qSr>)rrrr>r>r?rr*rpr1rPr)rrrr]z-infT)rr N) rrrJrrextendr%r$r r1r8r\r]rcrrrrrrrr! logical_or logical_notrrirrrrrr2)rr chunk_lengthrr flattenedrrwavformflattened_lensr r6rrr seq_rangelengths_expand padding_maskaudio_attention_mask_audio_attention_maskchunk_num_frame chunk_maskrrrrr7r'rr>r>r?get_audio_embeddingqs                     zMiniCPMO.get_audio_embeddingcCs|j||jjdd}|S)NF)rr( stream_input)get_omni_embeddingraudio_chunk_length)rr embeddingr>r>r?get_audio_features zMiniCPMO.get_audio_featureFcCs:|r||}n|||}t|}tjt|dd}|S)z Args: chunk_length: whisper use full attention or chunk attention stream_input: use streaming audio embedding Returns: final embeddings with audio feature rr2)rr3r$rJr7r)rrr(r4audio_embeddingsbs audio_embsr>r>r?r5s   zMiniCPMO.get_omni_embeddingcCstdd|D}tjtdd|Ddd}t||jdks"J|jjjjj }|jjjjj }dd|D}|dddf|dddf }t |tsRJtjjjj|dd d }|j\} } } |dd d| d d | }tj| d|ftj|d} |j| j d} | dddf| dddf}tj| d | j dd|dk| dddddf<|j||| |d}|||S)NcSg|]}|jqSr>rrr>r>r?rr#z.MiniCPMO.get_image_feature..cSr<r>)tgt_sizerr>r>r?rr#rr2cSs g|] }|jddddqS)r*)end_dimr)rrr/r>r>r?rr$r*Tg) batch_first padding_valuerprrr1rD)patch_attention_mask tgt_sizes)rrJr4r$r1rrposition_embeddingrr]r\r5rrrr utilsrnn pad_sequencerrTrMrNrrr8rrctyper)rr pixel_valuesrBr]r\all_pixel_values_lst max_patchesall_pixel_valuesBLrpatch_attn_masktgt_sizes_tensor mask_shapesvision_embeddingr>r>r?get_image_feature sB(     zMiniCPMO.get_image_feature positions forward_batchkwargscKst|||j||d}|S)N)r+rTlanguage_modelmultimodal_modelrS)rr)rr+rSrTrUrir>r>r?r4szMiniCPMO.forwardweightsc Csgd}t|}|D]\}}d|vsd|vrq d|vs!d|vr"q |jjrSd|vrSd|vs2d|vr?|dd }|dd }nd |vrS||vrS|d d }||vrS|}d |vr]|d d}|jjsfd|vrfq |jjssd|vsrd|vrsq |jjs|d |vr|q d|vsd|vsd|vrd|vsd|vrd|vr||}t|dt}|||q |D](\}} } | |vrq|| |}| dr||vrq||}|j }|||| n| dr||vrq ||}t|dt}|||q dS)N))qkv_projq_projq)rYk_projk)rYv_projr) gate_up_proj gate_projr)r_up_projr*zrotary_emb.inv_freq~r zrotary_emb.cos_cachedzrotary_emb.sin_cachedrz .weight_gz .weight_vz".parametrizations.weight.original0z".parametrizations.weight.original1z.weightrzself_attn.out_projzself_attn.projraudiosamplerrztts.model.layersz.mlp weight_loaderz.bias) dictnamed_parametersrrreplacerrgetattrr!endswithrd) rrXstacked_params_mapping params_dictr loaded_weight param_nameparamrd weight_nameshard_idr>r>r? load_weightsEsl        zMiniCPMO.load_weightsr)Nr)r)r)rF)&rFrGrHrrrrwrrstrr ModulerrrrrrrrJrKrrrr]rr!r3r8r5rRrrrrrrqrr>r>rr?rs*     G !|  ' $r)NNNrr*)rOrPrQr*T)ZrIr` dataclassesrtypingrrrrrrr numpyrvrJtorch.nn.functionalr rrtorch.nn.utils.parametrizerD parametrizerP torch.typestorch.nn.utilsr r transformersr rrrtransformers.activationsrtransformers.cache_utilsrrtransformers.modeling_outputsrr,transformers.models.whisper.modeling_whisperrrrsglang.srt.layers.quantizationrsglang.srt.managers.mm_utilsrr"sglang.srt.managers.schedule_batchrrr,sglang.srt.model_executor.forward_batch_inforsglang.srt.model_loader.utilsr $sglang.srt.model_loader.weight_utilsr!sglang.srt.models.idefics2r"sglang.srt.models.minicpmvr#r$sglang.srt.models.qwen2r&sglang.srt.utilsr'r(vector_quantize_pytorchr)rrrr@rArNrhrsrirrrrrrrr r EntryClassr>r>r>r?s $              $ L6-+Q_\