o ꁱiX@s&ddlmZmZmZmZddlZddlmZddlm Z m Z ddl m Z m Z ddlmZddlmZddlmZddlmZd d lmZmZd d lmZd d lmZmZeeZ e!ed rcej"durhgde_"GdddeZ#Gddde#Z$Gddde#eZ%e &ee$e &ee%gdZ'dS))ListOptionalTupleUnionN) AutoModelAutoModelForCausalLM)CausalLMOutputBaseModelOutputWithPast)modeling_utils)PreTrainedModel)logging)GenerationMixin) VibeVoiceTokenizerStreamingCacheVibeVoiceTokenizerEncoderOutput)VibeVoiceASRConfig)VibeVoiceCausalLMOutputWithPastSpeechConnectorALL_PARALLEL_STYLES)tpnonecolwiserowwisec@s@eZdZeZdZdZdZdZdZ dZ dZ dZ dZ dZddZdS)VibeVoiceASRPreTrainedModelmodelTpast_key_valuescCst|jdrt|jjdr|jjj}nt|jdr&t|jjdr&|jjj}nd}t|tjrF|jj j d|d|j durD|j j dSdSt|tj r[|jj d|j j dSdS)Nlanguage_model_configinitializer_rangedecoder_configg{Gz?gmeanstdg?)hasattrconfigrrr isinstancennLinearweightdatanormal_biaszero_ LayerNormfill_)selfmoduler!r0L/home/ubuntu/vibevoice-community/vibevoice/modular/modeling_vibevoice_asr.py _init_weights+s     z)VibeVoiceASRPreTrainedModel._init_weightsN)__name__ __module__ __qualname__r config_classbase_model_prefixsupports_gradient_checkpointing_skip_keys_device_placement_supports_cache_class_supports_flash_attn_supports_flash_attn_2_supports_sdpa_supports_quantized_cache_supports_static_cache_supports_attention_backendr2r0r0r0r1rs rcseZdZfddZddZddZddd Z          dd ejd e ej d e ejd e e e ej de ej de e de e de e de e de ejdee effddZZS)VibeVoiceASRModelcst|t|dr!|jdur!t|jtrtt|j}n|j}ntj}|j }t ||_ t |j ||_t |j||_t|j|j||_t|j|j||_dS)N torch_dtype)super__init__r"rBr$strgetattrtorchfloat32rr from_configlanguage_modelacoustic_tokenizer_configtoacoustic_tokenizersemantic_tokenizer_configsemantic_tokenizerracoustic_vae_dim hidden_sizeacoustic_connectorsemantic_vae_dimsemantic_connector)r.r#dtype lm_config __class__r0r1rD?s   zVibeVoiceASRModel.__init__cCsLt|jdr |jjS|jjD]\}}|jdkr!t|j|SqJd)N embed_tokenszembed_tokens.weightFzshould not arrive here)r"rJrYfullmapitems orig_namerF)r.nameattrr0r0r1get_input_embeddingsUs  z&VibeVoiceASRModel.get_input_embeddingscC ||j_dSN)rJrYr.valuer0r0r1set_input_embeddings_ z&VibeVoiceASRModel.set_input_embeddingsNcCs<||_||_|jdur|j|jdur|jdSdS)z@Set the speech tokenizers used for encoding and decoding speech.N)rMrOeval)r.rMrOr0r0r1set_speech_tokenizersbs   z'VibeVoiceASRModel.set_speech_tokenizers input_idsattention_mask position_idsr inputs_embeds use_cacheoutput_attentionsoutput_hidden_states return_dictcache_positionreturnc KsZ| dur| n|jj} |jd||||||||| | d | } | s!| St| j| j| j| jdS)N rhrirjrrkrlrmrnrorp)last_hidden_stater hidden_states attentionsr0)r#use_return_dictrJr rsrrtru) r.rhrirjrrkrlrmrnrorpkwargsoutputsr0r0r1forwardns. zVibeVoiceASRModel.forward)NN) NNNNNNNNNN)r3r4r5rDr_rdrgrG LongTensorrTensorr FloatTensorboolrr ry __classcell__r0r0rWr1rA>sL      rAc#s|eZdZdZdgZddiZfddZddZd d Zd d Z d dZ ddZ ddZ ddZ   d/dejdeejdeejdefddZ               d0deejdeejdeejd eeejd!eejd"eejd#eed$eed%eed&eed'eejdeejdeejdeejd(eejd)eeeff d*d+Z      ,    d1d-d.ZZS)2$VibeVoiceASRForConditionalGenerationz VibeVoice model for Automatic Speech Recognition (ASR) with language modeling head for conditional generation. This class is designed for inference and generation tasks. zlm_head.weightlm_head colwise_repcst|t||_|jj|_t|dr+|jdur+t|jt r't t |j}n|j}nt j }t j|jj|jdd||_|dS)NrBF)r*)rCrDrArr vocab_sizer"rBr$rErFrGrHr%r&rQrLr post_init)r.r#rUrWr0r1rDs     z-VibeVoiceASRForConditionalGeneration.__init__cCs |jSra)rr_r.r0r0r1r_ z9VibeVoiceASRForConditionalGeneration.get_input_embeddingscCs|j|dSra)rrdrbr0r0r1rdsz9VibeVoiceASRForConditionalGeneration.set_input_embeddingscCs|jSrarrr0r0r1get_output_embeddingssz:VibeVoiceASRForConditionalGeneration.get_output_embeddingscCs ||_dSrar)r.new_embeddingsr0r0r1set_output_embeddingsrz:VibeVoiceASRForConditionalGeneration.set_output_embeddingscCr`rarrJ)r.decoderr0r0r1 set_decoderrez0VibeVoiceASRForConditionalGeneration.set_decodercCs|jjSrarrr0r0r1 get_decodersz0VibeVoiceASRForConditionalGeneration.get_decodercCsDt|jjddr |}|}t|dr|j|_dS||_dSdS)zGTie the weights between the input embeddings and the output embeddings.tie_word_embeddingsFr'N)rFr#rrr_r"r')r.output_embeddingsinput_embeddingsr0r0r1 tie_weightss   z0VibeVoiceASRForConditionalGeneration.tie_weightsNN@speech_tensors speech_masksspeech_semantic_tensorsstreaming_segment_durationc! Cst|jdr |jjdur t|jjtrtt|jj}n|jj}ntj}||}|j dkr2| d}|j \}}d}t ||} || k} t | s|jj| d} | j|jjjdd} |j| } |durq|j|}n|jj| dj}|j|}nt}t}g}g}tj||jd}dt d t fd d }t||| }t|}t|D]I\}\}}|dd||f}|dkrq||dk}|jjj| d||d |d }||j|jjj| d||d |d }||jqtj |dd}t!||jjj"d}|j|jjjdd} |j| } tj |dd}|j|}|dur7| |||} n | |} Wd| SWd| S1sOwY| S)a Encode speech input into features that can be used by the language model. This method is called once before generation to process the speech input. For long audio (>600s by default), uses streaming processing to avoid conv overflow (>2^32). Segments are processed independently, then concatenated before final sampling. Args: speech_tensors: Input audio tensor [batch_size, samples] speech_masks: Optional mask for speech features speech_semantic_tensors: Optional pre-computed semantic tokens streaming_segment_duration: Segment duration in seconds for streaming processing (default: 60s) rBNrri]) dist_typedevice total_lengthsegment_lengthcssH|dkr tdtd||D]}t|||}||kr!||fVqdS)z8Iterate over audio segments with a given segment length.rzsegment_length must be positiveN) ValueErrorrangemin)rrstartendr0r0r1_iter_segmentss zJVibeVoiceASRForConditionalGeneration.encode_speech.._iter_segmentsT)cachesample_indicesrlis_final_chunk)dimr)#r"r#rBr$rErFrGrHrLndim unsqueezeshapeintno_gradrrMencodesample std_dist_typerRrTrOr rarangerlistlen enumerate contiguousnumelappendcatrfix_std)!r.rrrrrU batch_size total_samples sample_ratesegment_samples use_streamingencoder_output audio_tokensacoustic_featuressemantic_featuressemantic_tokensacoustic_encoder_cachesemantic_encoder_cacheacoustic_mean_segmentssemantic_mean_segmentsrrsegments num_segmentsseg_idxrrchunkis_finalacoustic_encoder_outputsemantic_encoder_outputacoustic_mean_fullcombined_featuresr0r0r1 encode_speechs                 X XXz2VibeVoiceASRForConditionalGeneration.encode_speechrhrirjrrklabelsrlrmrnrorpacoustic_input_maskrqc Ks|dur|n|jj}| dur| n|jj} | dur| n|jj} |dur$|n|jj}|dur6|dur6||}| durJ|durJ|j| | |d}|||<|jd||||||| | | d }| s_|dn|j}| |}d}|dur|dddddf }|dddf }t }| d|j}| d}||j}|||}| s|f|dd}|dur|f|S|St|||j|j|jdS) z] Forward pass for the model. Handles both training and generation scenarios. N)rrrrrr.r)losslogitsrrtru)r#rmrnrvrlr_rrrsrrr%CrossEntropyLossviewrrLrrrrtru)r.rhrirjrrkrrlrmrnrorprrrrrwspeech_featuresrxrtrr shift_logits shift_labelsloss_fctoutputr0r0r1ryUs\     z,VibeVoiceASRForConditionalGeneration.forwardTc Ks|dur,t|tr|ddjd} n|} |dur,|jd| kr,|dd| df}|dur[|dur[|dd}||dkd|dur[|dur[|dd|jd df}|dur|durg|nd}tj|||durv|jdn|jd|dur|j n|j d}|dur|durd|i}nd|i}| |||||d |durt |dkr|ddkr| || | | d n | ddddd | | |S) a= Prepare inputs for generation step. This method is called by generate() for each token generation step. Following Qwen2-VL's approach: speech inputs are only forwarded on the first pass (when cache_position[0] == 0), and are excluded in subsequent generation steps. Nrrrrrkrh)rjrprrlri)rrrr) r$tuplerget_seq_lengthlongcumsum masked_fill_rGrrupdater)r.rhrrirkrprjrlrrrrrw past_lengthpast_seen_tokens model_inputsr0r0r1prepare_inputs_for_generationsX     zBVibeVoiceASRForConditionalGeneration.prepare_inputs_for_generation)NNr)NNNNNNNNNNNNNNN) NNNNNTNNNN)r3r4r5__doc___tied_weights_keys_tp_planrDr_rdrrrrrrGr|r BoolTensorfloatrrzr{rr}rrrryrr~r0r0rWr1rs          Vr)rrAr)(typingrrrrrGtorch.nnr%transformers.models.autorrtransformers.modeling_outputsrr transformersr transformers.modeling_utilsr transformers.utilsr transformers.generationr modular_vibevoice_tokenizerrrconfiguration_vibevoicermodeling_vibevoicerr get_loggerr3loggerr"rrrArregister__all__r0r0r0r1s.         Z k