o c ir@spdZddlZddlmZddlmZmZmZmZddl Z ddl Z ddl Z ddlmZeeZGdddZdS) z FasterQwen3TTS: Real-time TTS using CUDA graph capture. Wrapper class that provides a Qwen3-TTS API while using CUDA graphs for 6-10x speedup. N)Path) GeneratorOptionalTupleUnion)suppress_flash_attn_warningc(@seZdZdZdejdfdedejdefddZ e dejd dfd edede eejfd edef d d Z defddZ      dFdedededededededeeeffddZdGd!e eefd"edeejeffd#d$Z  % dHded!e eefd&eded'ed(ed)efd*d+Z ,dIdeded-eed.eefd/d0Z ,dId(efd1d2Ze  3   4     dJdeded!e eefd&eded5ededed6ededed'ed(ed)edeeeffd7d8Ze  3   4   9    %dKdeded!e eefd&eded5ededed6ededed:ed'ed(ed)ed;edeeejeefd,d,ff"dd?Z e ,  3   4   9dMded-eded.eeded5ededed6ededed:edeeejeefd,d,ffd@dAZ!e  3   4  dNded.ededed5ededed6edededeeeffdBdCZ"e  3   4   9dOded.ededed5ededed6ededed:edeeejeefd,d,ffdDdEZ#d,S)PFasterQwen3TTSz Qwen3-TTS model with CUDA graphs for real-time inference. Compatible API with Qwen3TTSModel, but uses CUDA graph capture for 6-10x speedup on NVIDIA GPUs. cudadevicedtype max_seq_lencCs:||_||_||_||_||_||_d|_d|_i|_dS)Ni.F) modelpredictor_graph talker_graphr r r sample_rate _warmed_up_voice_prompt_cache)self base_modelrrr r rrL/home/ubuntu/vllm_env/lib/python3.10/site-packages/faster_qwen3_tts/model.py__init__s  zFasterQwen3TTS.__init__sdpa model_nameattn_implementationc Cst|tr tt|}|drtjstdt d|t ddl m }Wdn1s4wYddl m}dd lm}|j||||d } | jj} | jjj} | j} | jj} | j}t d || | |||d d dd}|| j| |||d}t d|| |||||dS)a Load Qwen3-TTS model and prepare CUDA graphs. Args: model_name: Model path or HuggingFace Hub ID device: Device to use ("cuda" or "cpu") dtype: Data type for inference attn_implementation: Attention implementation ("sdpa" or "flash_attention_2") max_seq_len: Maximum sequence length for static cache Returns: FasterQwen3TTS instance r zCUDA graphs require CUDA devicezLoading Qwen3-TTS model: r) Qwen3TTSModelNr)PredictorGraph) TalkerGraph) device_map torch_dtyperzBuilding CUDA graphs...T2?)r r do_sampletop_k temperature)r r rz3CUDA graphs initialized (will capture on first run))rrrr r r) isinstancestrgetattrtorch startswithr is_available ValueErrorloggerinforqwen_ttsrrrrrfrom_pretrainedrtalkerconfig talker_configcode_predictor hidden_size)clsrr r rrrrrrr2r4 predictor pred_config talker_hiddenrrrrrr11s`        zFasterQwen3TTS.from_pretrained prefill_lencCsF|jrdStd|jjdd|jj|ddd|_tddS)z:Warm up and capture CUDA graphs with given prefill length.NzWarming up CUDA graphs...) num_warmup)r;r=TzCUDA graphs captured and ready)rr.r/rcapturer)rr;rrr_warmups zFasterQwen3TTS._warmupEnglishr#r"T?textlanguagemax_new_tokensr&r%r$repetition_penaltyreturncCstd)z Generate speech from text using default voice. Not yet implemented - use generate_voice_clone() instead. z^Default voice generation not yet implemented. Use generate_voice_clone() with reference audio.)NotImplementedError)rrBrCrDr&r%r$rErrrgenerateszFasterQwen3TTS.generate? ref_audio silence_secscCsdtjt|ddd\}}|jdkr|jdd}|dkr.tjt||tjd}t ||g}||fS)aLoad reference audio and optionally append trailing silence. The ICL voice-cloning prompt ends with the last codec token of the reference audio, so the model's first generated token is conditioned on whatever phoneme the reference ends with. Appending a short silence makes the last tokens encode silence instead, preventing that phoneme from bleeding into the start of the generated speech. Set silence_secs=0 to disable this behavior. float32F)r always_2dr)axisrr ) sfreadr(ndimmeannpzerosintrL concatenate)rrJrKaudiosrsilencerrr_load_ref_audio_with_silences  z+FasterQwen3TTS._load_ref_audio_with_silenceFref_text xvec_onlynon_streaming_modeappend_silencec Cs|j|g}|j|} t||||f} | |jvr"|j| \} } np|rO|jjt|ddd} | dj}tdg|gdgdgd} dgt| } | | f|j| <nC|rSdnd }|j ||d }|jj||d } |j | } g} | dj }|r|j |g}| |j|dn| d| | f|j| <|jj}|j|| | | |dur|gnd gd|d \}}}}|js||jd|j}|jj}d|_d}|s| dr| dddur| dd}||||||||fS)aPrepare inputs for generation (shared by streaming and non-streaming). Args: xvec_only: When True (default), use only the speaker embedding (x-vector) for voice cloning instead of the full ICL acoustic prompt. This prevents the model from continuing the reference audio's last phoneme and allows natural language switching. When False, the full reference audio codec tokens are included in context (ICL mode). T)rJr\x_vector_only_moderNF)ref_coderef_spk_embeddingraicl_moderI)rK)rJr\Auto)m input_idsref_idsvoice_clone_prompt languagesspeakersr^rrb)r_build_assistant_text_tokenize_textsr(rcreate_voice_clone_promptrcdictlenr[#_prompt_items_to_voice_clone_promptr\_build_ref_textappend_build_talker_inputs_localrr?shaper2r3r4 rope_deltasget)rrBrJr\rCr]r^r_ input_textsrh cache_keyvcpri prompt_itemsspk_embrKref_audio_inputrt ref_textsrgtietamtthtper2r3 ref_codesrrr_prepare_generationsj         z"FasterQwen3TTS._prepare_generationNspeakerinstructc Cs|j|g}|j|}g}|dus|dkr|dn||j|j|gd|jj}|j||dgd|dur?|gndg|gd|d\} } } } |jsW|| jd|j } |j j }d| _ || || | | | fS)Nr`rrfF)rgrhrirjrkrlr^ instruct_idsr) rrmrnrt_build_instruct_textrurr?rvr2r3r4rw)rrBrCrrryrhrrgrrrrr2r3rrr_prepare_generation_customs.    z)FasterQwen3TTS._prepare_generation_customc 1 Csddtt|D} d} |dur||} |dur7t|D]\} } | dur6| | |j|j| q|durBdgt|}g} d}tt|||D]i\} \}}}| dur|dksb|dured}n>| |j j j vrvt d|d|j j j | }|jtj||jj|jd}n|d| s|d | r| | }nd}|dusJ| d krd}n| |j j jvrt d |d|j j j| }| d vr|d vr|j j j| r|j j j| }|j j j|}|j|jtj|j j|j j|j jgg|jj|jdjddd\}}}|dur*|j j j|j j j|j j jgg}n|j j j|j j j||j j jgg}|jtj||jj|jd}|jtj|j j j|j j jgg|jj|jd}|durrtj||gdd}ntj|| ddd|gdd}|j|j|ddddf}tj|!d|j"ddd|fdd|ddddf}tj||fdd}|dur|#dddur|d | r|j$|ddddf|| ddddf|d| %|jj&|||d\}} tj||gdd}ntj||j|j|ddddf|ddddfgdd}|r|ddddf}tj|tj|j|j|ddddf|fdd|jtj|j j jg|ddddfj"ddg|jj|jd||jtj|j j jgg|jj|jdgdd}|} ntj|j|j|ddddf|fdd} | | || | qNt| D]\} }tjdd|Ddd| | <qtdd| D}!dd| D}"dd|"D}#tj'j(j)j*|#ddd}$|$j+dgd} | j"d | j"d}%}&t,|&!|%d}'|&|!}(|'|(-dk.%| j})|/}*d!d| D}+d"d|+D},tj'j(j)j*|+ddd}-tj,t0|,|-jd#!t|,d}.tj|,|-jd#-d}/|.|/k}0|*|-|0<|-} | |)| |fS)$zDLocal copy of upstream talker input building for qwen-tts main repo.cSsg|]}gqSrr).0_rrr 9sz=FasterQwen3TTS._build_talker_inputs_local..Nr`zSpeaker z not implemented)r r rardautoz Language )chineser)r`Nr<rdimrb)text_idref_idrb tts_pad_embed tts_eos_embedr^cSsg|]}|dur|qSNr)ritemrrrrscSg|]}|jdqS)rrvrtrrrrcSg|]}|dqSrsqueezerrrrrrcSsg|] }|jdgdqS)rdims)fliprrrrrsTre) batch_first padding_valuerrcSrrrrrrrrrcSrrr)rsrrrrr)r )1rangerqgenerate_speaker_prompt enumeratertr2text_projectionget_text_embeddingsziplowerr3r4spk_idrGget_input_embeddingsr*tensorr r codec_language_idspk_is_dialecttts_bos_token_idtts_eos_token_idtts_pad_token_idchunkcodec_nothink_idcodec_think_bos_idcodec_think_eos_idcodec_think_id codec_pad_id codec_bos_idcatviewexpandrvrxgenerate_icl_prompttoclonennutilsrnn pad_sequencerarange unsqueezelongrmax)1rrgrhrirjrkrlr^rtalker_input_embedsvoice_clone_spk_embedsindex instruct_idtrailing_text_hiddensrinput_idrCr speaker_embedr language_iddialect tts_bos_embedrcodec_prefill_listcodec_input_emebdding_0codec_input_emebdding_1codec_input_emebdding_talker_input_embed_role_talker_input_embedtalker_input_embedicl_input_embedtrailing_text_hiddenoriginal_lengths sequencessequences_reversedpadded_reversed batch_sizemax_lenindicesnum_padstalker_attention_maskpad_embedding_vectorsequences_to_padtrailing_text_original_lengthspadded_hiddens arange_tensorlengths_tensor padding_maskrrrru-sn              *      z)FasterQwen3TTS._build_talker_inputs_localr?min_new_tokenstop_pc(Csddlm}|j||||| | |d\}}}}}}}}||||||||j|j||||| | | d\}}|durFtdtjdtj dg|j fS|j }|dur]| |j }tj||gdd }n|}|d |di\}}|durt|jdnd}|jd} g}!|D]7}"t|"d r|"}"n t|"d r|"n|"}"|dkrt|t| dt|"}#|"|#d}"|!|"q|d }$|$d}%|dd|d}&|&dkr|%|&nd}'td|%dd|&dd|ddd|'dd |!|fS)a9 Generate speech with voice cloning using reference audio. Args: text: Text to synthesize language: Target language ref_audio: Path to reference audio file ref_text: Transcription of reference audio max_new_tokens: Maximum tokens to generate min_new_tokens: Minimum tokens before EOS is allowed temperature: Sampling temperature top_k: Top-k sampling top_p: Top-p (nucleus) sampling do_sample: Whether to sample repetition_penalty: Repetition penalty xvec_only: When True (default), use only the speaker embedding for voice cloning. This prevents phoneme bleed-through from the reference and allows clean language switching. Set to False for full ICL mode (reference audio in context). non_streaming_mode: Match upstream non-streaming prompt layout. Default True for better non-streaming quality. Returns: Tuple of ([audio_waveform], sample_rate) r fast_generaterCr]r^r_r2rattention_maskrrr3rrrDrr&r%rr$rENGeneration returned no tokensrOrr audio_codescpuflattensteps(@ prefill_msdecode_s Generated .2f s audio in s ( ms_per_step.1fms/step, RTF: ))rHrrrrr.warningrTrUrLrspeech_tokenizerrr r*rdecoderrvhasattrrrnumpyrVrrqrtr/)(rrBrCrJr\rDrr&r%rr$rEr]r^r_rrgr2r3rrrrr codec_idstimingr ref_codes_devcodes_for_decode audio_listrYref_len total_len audio_arraysacutn_stepsaudio_duration total_timertfrrrgenerate_voice_clone sv )        z#FasterQwen3TTS.generate_voice_clone chunk_size parity_modec5csddlm}m}|j||||| ||d\}}}}}}}}|j}d}t|| }g}d}d} |r1|n|}!t||||||||||| | | | d}"|sQ|j|"d<|j|"d <|!di|"D]\}#}$| |#|#j d}%t j |dd }&|&j d}'| dur|durt j | |&j|&gdd }(n|&}(|d |(di\})}*|)d}+t|+d r|+}+n t|+d r|+n|+}+|dur|j d},|(j d}-t|,t|-dt|+}.|+|.d}/n|+}/|/|d}0t|/}|'|krt|/|'} nWtd|'|%|}1|&|1d}2|2j d|%}3|d |2di\})}*|)d}+t|+d r!|+}+n t|+d r+|+n|+}+|3dkrAtt|3| }4|+|4d}0n|+}0|0|*|$fVqXdS)a Stream voice-cloned speech generation, yielding audio chunks. Same as generate_voice_clone() but yields (audio_chunk, sample_rate, timing) tuples every chunk_size codec steps (~chunk_size/12 seconds of audio). Args: text: Text to synthesize language: Target language ref_audio: Path to reference audio file ref_text: Transcription of reference audio max_new_tokens: Maximum tokens to generate min_new_tokens: Minimum tokens before EOS is allowed temperature: Sampling temperature top_k: Top-k sampling top_p: Top-p (nucleus) sampling do_sample: Whether to sample repetition_penalty: Repetition penalty chunk_size: Codec steps per chunk (12 = ~1 second) xvec_only: When True (default), use only the speaker embedding for voice cloning. This prevents phoneme bleed-through from the reference and allows clean language switching. Set to False for full ICL mode (reference audio in context). non_streaming_mode: When True (default), prefill the full target text before streaming decode. Set to False to feed text token-by-token during decode. parity_mode: When True, disables CUDA graphs and uses dynamic cache streaming. Yields: Tuple of (audio_chunk_numpy, sample_rate, timing_dict) r)fast_generate_streamingparity_generate_streamingrrN)r2rrrrr3rDrr&r%rr$rEr"rrrrrrr) streamingr$r%rrrrprrrtrvr*rrr rrrrrrrVrqround)5rrBrCrJr\rDrr&r%rr$rEr"r]r^r_r#r$r%rgr2r3rrrrrrcontext_framesmin_calibration_frames all_codesprev_gen_audio_lensamples_per_frame stream_fn stream_kwargs codec_chunkrn_newall_flatn_total codes_inputrrYrXrr ref_audio_cut gen_audio new_audio ctx_startwindown_ctx ctx_samplesrrrgenerate_voice_clone_streaming|s1                  z-FasterQwen3TTS.generate_voice_clone_streamingc Cs|jjjdkr td|j|g|j|g|jjjdvr"d}ddlm} |j||||d\} }}}}}}| |||||||j |j ||||| | | d\}}|durdt dt jdt jd g|jfS| j}|d |d i\}}g}|D]!}t|d r||qw|t|d r|n|qw|d}|d}|dd|d}|d kr||nd }t d|dd|dd|ddd|dd ||fS)N custom_voice5Loaded model does not support custom voice generation0b6rrrBrCrrrrrOrrrrrrrrrrrrrr r r r )rtts_model_typer-_validate_languages_validate_speakerstts_model_sizerHrrrrr.r rTrUrLrrrrrrtrrrr/)rrBrrCrrDrr&r%rr$rErrgr2r3rrrrrrrrrYrrrrrrrrrgenerate_custom_voicesh    z$FasterQwen3TTS.generate_custom_voicec (csx|jjjdkr td|j|g|j|g|jjjdvr#d}ddlm} |j||||d\}}}}}}}|j }d}t || }g}d}d}| did |d |d |d |d |d|d|j d|j d|d|d|d|d| d| d| d| D]\}}| ||jd}tj|dd}|jd}|dur|d|di\} }!| d}"t|"dr|"}"n t|"dr|"n|"}"|"|d}#t|"}||krt|"|}nWt d|||}$||$d}%|%jd|}&|d|%di\} }!| d}"t|"dr|"}"n t|"dr|"n|"}"|&dkr1tt|&|}'|"|'d}#n|"}#|#|!|fVqdS)Nr=r>r?rr$r@r&rr2rrrrr3rrrDrr&r%rr$rEr"rrrrr)rrAr-rBrCrDr'r$rrrrrrtrvr*rrrrrrrrqrVr()(rrBrrCrrDrr&r%rr$rEr"r$rgr2r3rrrrrr)r*r+prev_audio_lenr-r0rr1r2r3rrYrXr7r8r9r:r;rrrgenerate_custom_voice_streamingcs                z.FasterQwen3TTS.generate_custom_voice_streamingc Cs|jjjdkr td|j|gddlm} |j||d|d\} } }}}}}| | ||||||j|j|||||| | d\}}|durTt dt j dt j dg|jfS| j}|d |d i\}}g}|D]!}t|d rz||qg|t|d r|n|qg|d }|d}|dd|d}|d kr||nd }t d|dd|dd|ddd|dd ||fS)N voice_design5Loaded model does not support voice design generationrrr@rrrOrrrrrrrrrrrrrr r r r )rrAr-rBrHrrrrr.r rTrUrLrrrrrrtrrrr/)rrBrrCrDrr&r%rr$rErrgr2r3rrrrrrrrrYrrrrrrrrrgenerate_voice_designsb    z$FasterQwen3TTS.generate_voice_designc 'csX|jjjdkr td|j|gddlm} |j||d|d\} }}}}}}| j}d}t|| }g}d}d}| did|d |d |d |d |d |d|j d|j d|d|d|d|d|d| d| d| D]\}}| ||j d}t j|dd}|j d}|dur|d|di\}} |d}!t|!dr|!}!n t|!dr|!n|!}!|!|d}"t|!}||krt|!|}nWtd|||}#||#d}$|$j d|}%|d|$di\}} |d}!t|!dr|!}!n t|!dr |!n|!}!|%dkr!tt|%|}&|!|&d}"n|!}"|"| |fVqpdS)NrIrJrrFr@r&rr2rrrrr3rrrDrr&r%rr$rEr"rrrrr)rrAr-rBr'r$rrrrrrtrvr*rrrrrrrrqrVr()'rrBrrCrDrr&r%rr$rEr"r$rgr2r3rrrrrr)r*r+rGr-r0rr1r2r3rrYrXr7r8r9r:r;rrrgenerate_voice_design_streaming s                z.FasterQwen3TTS.generate_voice_design_streaming)r@r r#r"TrA)rI)TFTr) r rr#r"rTrATTT) r rr#r"rTrAr!TTTF)Nr rr#r"rTrA) Nr rr#r"rTrAr!)r rr#r"rTrA)r rr#r"rTrAr!)$__name__ __module__ __qualname____doc__r*bfloat16r(r rVr classmethodrr1r?floatboolrlistrHrrTndarrayr[rrrruinference_moder rrpr<rErHrKrLrrrrr sr    R *  Y . a        n            L      a    G     r )rPloggingpathlibrtypingrrrrrrT soundfilerPr*rr getLoggerrMr.r rrrrs