o %ݫixy@sdZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZddlmZdd lmZeeZGd d d e ZGd d d e ZGddde ZGddde ZdS)aR Specifies the inference interfaces for Text-To-Speech (TTS) modules. Authors: * Aku Rouhe 2021 * Peter Plantinga 2021 * Loren Lugosch 2020 * Mirco Ravanelli 2020 * Titouan Parcollet 2021 * Abdel Heba 2021 * Andreas Nautsch 2022, 2023 * Pooneh Mousavi 2023 * Sylvain de Langen 2023 * Adel Moumen 2023 * Pradnya Kandarkar 2023 N)EncoderClassifier)MelSpectrogramEncoder) Pretrained)GraphemeToPhoneme)fetch) get_logger)text_to_sequencecsHeZdZdZddgZfddZddZdd Zd d Zd d Z Z S) Tacotron2a A ready-to-use wrapper for Tacotron2 (text -> mel_spec). Arguments --------- *args : tuple **kwargs : dict Arguments are forwarded to ``Pretrained`` parent class. Example ------- >>> tmpdir_tts = getfixture('tmpdir') / "tts" >>> tacotron2 = Tacotron2.from_hparams(source="speechbrain/tts-tacotron2-ljspeech", savedir=tmpdir_tts) >>> mel_output, mel_length, alignment = tacotron2.encode_text("Mary had a little lamb") >>> items = [ ... "A quick brown fox jumped over the lazy dog", ... "How much wood would a woodchuck chuck?", ... "Never odd or even" ... ] >>> mel_outputs, mel_lengths, alignments = tacotron2.encode_batch(items) >>> # One can combine the TTS model with a vocoder (that generates the final waveform) >>> # Initialize the Vocoder (HiFIGAN) >>> tmpdir_vocoder = getfixture('tmpdir') / "vocoder" >>> from speechbrain.inference.vocoders import HIFIGAN >>> hifi_gan = HIFIGAN.from_hparams(source="speechbrain/tts-hifigan-ljspeech", savedir=tmpdir_vocoder) >>> # Running the TTS >>> mel_output, mel_length, alignment = tacotron2.encode_text("Mary had a little lamb") >>> # Running Vocoder (spectrogram-to-waveform) >>> waveforms = hifi_gan.decode_batch(mel_output) modelrcs4tj|i|t|jddg|_|jjj|_dS)N text_cleanersenglish_cleaners)super__init__getattrhparamsr r inferselfargskwargs __class__M/home/ubuntu/.local/lib/python3.10/site-packages/speechbrain/inference/TTS.pyrFs  zTacotron2.__init__cCs|j||j}|t|fSzHEncodes raw text into a tensor with a customer text-to-sequence function)rrr lenrtxtsequencerrr text_to_seqMs zTacotron2.text_to_seqcst@fdd|D}tjj|}fdd|D}|t|ddks*Jdtj|jd} |j j |\}}}Wdn1sGwY|||fS) aNComputes mel-spectrogram for a list of texts Texts must be sorted in decreasing order on their lengths Arguments --------- texts: List[str] texts to be encoded into spectrogram Returns ------- tensors of output spectrograms, output lengths and alignments c*g|]}dtj|djdiqStext_sequencesrdevice)torchtensorrr$.0itemrrr az*Tacotron2.encode_batch..csg|] }|dqS))rr'r*rrr+ksTreverse0input lengths must be sorted in decreasing orderr#N) r%no_grad speechbraindataiobatch PaddedBatchsortedr&r$rr"data)rtextsinputslens input_lengthsmel_outputs_postnet mel_lengths alignmentsrr*r encode_batchRs"     zTacotron2.encode_batchcCs ||gS)z$Runs inference for a single text strr?)rtextrrr encode_textvs zTacotron2.encode_textcCs ||S)zEncodes the input texts.r@)rr8rrrforwardzs zTacotron2.forward) __name__ __module__ __qualname____doc__HPARAMS_NEEDEDrrr?rBrC __classcell__rrrrr #s $r csNeZdZdZdgZfddZddZddZd d Zd d Z d dZ Z S) MSTacotron2a A ready-to-use wrapper for Zero-Shot Multi-Speaker Tacotron2. For voice cloning: (text, reference_audio) -> (mel_spec). For generating a random speaker voice: (text) -> (mel_spec). Arguments --------- *args : tuple **kwargs : dict Arguments are forwarded to ``Pretrained`` parent class. Example ------- >>> tmpdir_tts = getfixture('tmpdir') / "tts" >>> mstacotron2 = MSTacotron2.from_hparams(source="speechbrain/tts-mstacotron2-libritts", savedir=tmpdir_tts) # doctest: +SKIP >>> # Sample rate of the reference audio must be greater or equal to the sample rate of the speaker embedding model >>> reference_audio_path = "tests/samples/single-mic/example1.wav" >>> input_text = "Mary had a little lamb." >>> mel_output, mel_length, alignment = mstacotron2.clone_voice(input_text, reference_audio_path) # doctest: +SKIP >>> # One can combine the TTS model with a vocoder (that generates the final waveform) >>> # Initialize the Vocoder (HiFIGAN) >>> tmpdir_vocoder = getfixture('tmpdir') / "vocoder" >>> from speechbrain.inference.vocoders import HIFIGAN >>> hifi_gan = HIFIGAN.from_hparams(source="speechbrain/tts-hifigan-libritts-22050Hz", savedir=tmpdir_vocoder) # doctest: +SKIP >>> # Running the TTS >>> mel_output, mel_length, alignment = mstacotron2.clone_voice(input_text, reference_audio_path) # doctest: +SKIP >>> # Running Vocoder (spectrogram-to-waveform) >>> waveforms = hifi_gan.decode_batch(mel_output) # doctest: +SKIP >>> # For generating a random speaker voice, use the following >>> mel_output, mel_length, alignment = mstacotron2.generate_random_voice(input_text) # doctest: +SKIP r cstj|i|dg|_|jjj|_|jj|_tj|jj d|j id|_ d|_ |jr:t j|jj d|j id|_ dSt j|jj d|j id|_ dS)Nr r$)run_opts)sourcerK)r rr rr rcustom_mel_spec_encoderr from_hparamsg2pr$spk_emb_encoderrrrrrrrs"    zMSTacotron2.__init__cCst||j}|t|fSr)rr rrrrr __text_to_seqs  zMSTacotron2.__text_to_seqc Cst|\}}||jjkrtj|||jj}||j}|jr'|j |}n|j |}| d}t |tr:|g}||}tt|D]}d||||<d||d||<qE|t|d}|||S)aD Generates mel-spectrogram using input text and reference audio Arguments --------- texts : str or list Input text audio_path : str Reference audio Returns ------- tensors of output spectrograms, output lengths and alignments r {}r-) torchaudioloadrspk_emb_sample_rate functionalresampletor$rMrPencode_waveformr?squeeze isinstancestrrOrangerjoinrepeat_MSTacotron2__encode_batch) rr8 audio_path ref_signal signal_srspk_emb phoneme_seqsispk_embsrrr clone_voices$        zMSTacotron2.clone_voicecCs|}||j}t|tr|g}||}tt|D]}d ||||<d||d||<q| t|d}| ||S)a Generates mel-spectrogram using input text and a random speaker voice Arguments --------- texts : str or list Input text Returns ------- tensors of output spectrograms, output lengths and alignments rRrSrTr-) #_MSTacotron2__sample_random_speakerfloatrZr$r]r^rOr_rr`rarb)rr8rfrgrhrirrrgenerate_random_voices     z!MSTacotron2.generate_random_voicec stHfdd|D}t|dddd}dd|D}tjj|}|t|ddks1Jd tj|jd } |j j ||\}}}Wd n1sOwY|||fS) aComputes mel-spectrograms for a list of texts Texts are sorted in decreasing order on their lengths Arguments --------- texts: List[str] texts to be encoded into spectrogram spk_embs: torch.Tensor speaker embeddings Returns ------- tensors of output spectrograms, output lengths and alignments cr r!)r%r&_MSTacotron2__text_to_seqr$r'r*rrr+r,z.MSTacotron2.__encode_batch..cSs|ddS)Nr"rsize)xrrr(sz,MSTacotron2.__encode_batch..T)keyr/cSsg|] }|ddqS)r"rro)r(entryrrrr+,sr.r0r#N) r%r1r6r2r3r4r5r&r$rr"r7) rr8rir9r:r;r<r=r>rr*r__encode_batch s,       zMSTacotron2.__encode_batchc st|jj|jj|jjjd}t|}|d}|d}|d}t|}d|t d|d<tj d|j d}t ||dkD])}tjj||||tfdd tt||D} tj|| fdd }q?|S) zSamples a random speaker embedding from a pretrained GMM Returns ------- x: torch.Tensor A randomly sampled speaker embedding )filenamerLsavedirgmm_n_components gmm_meansgmm_covariancesr-rr#csg|]}qSr)sample)r(_d_krrr+Yz7MSTacotron2.__sample_random_speaker..)dim)rrrandom_speaker_samplerrandom_speaker_sampler_source pretrainer collect_inr%rVzerosrandomrandintemptyr$arange distributionsmultivariate_normalMultivariateNormalstackr_intcat) rspeaker_gmm_local_pathrandom_speaker_gmmrxryrzcountsrqkx_krr}r__sample_random_speaker:s&    $z#MSTacotron2.__sample_random_speaker) rDrErFrGrHrrnrjrmrbrkrIrrrrrJs 0.rJcsTeZdZdZgdZfddZdddZ ddd Z dd d Zdd d Z Z S) FastSpeech2a A ready-to-use wrapper for Fastspeech2 (text -> mel_spec). Arguments --------- *args : tuple **kwargs : dict Arguments are forwarded to ``Pretrained`` parent class. Example ------- >>> tmpdir_tts = getfixture('tmpdir') / "tts" >>> fastspeech2 = FastSpeech2.from_hparams(source="speechbrain/tts-fastspeech2-ljspeech", savedir=tmpdir_tts) # doctest: +SKIP >>> mel_outputs, durations, pitch, energy = fastspeech2.encode_text(["Mary had a little lamb."]) # doctest: +SKIP >>> items = [ ... "A quick brown fox jumped over the lazy dog", ... "How much wood would a woodchuck chuck?", ... "Never odd or even" ... ] >>> mel_outputs, durations, pitch, energy = fastspeech2.encode_text(items) # doctest: +SKIP >>> >>> # One can combine the TTS model with a vocoder (that generates the final waveform) >>> # Initialize the Vocoder (HiFIGAN) >>> tmpdir_vocoder = getfixture('tmpdir') / "vocoder" >>> from speechbrain.inference.vocoders import HIFIGAN >>> hifi_gan = HIFIGAN.from_hparams(source="speechbrain/tts-hifigan-ljspeech", savedir=tmpdir_vocoder) # doctest: +SKIP >>> # Running the TTS >>> mel_outputs, durations, pitch, energy = fastspeech2.encode_text(["Mary had a little lamb."]) # doctest: +SKIP >>> # Running Vocoder (spectrogram-to-waveform) >>> waveforms = hifi_gan.decode_batch(mel_outputs) # doctest: +SKIP ) spn_predictorr input_encodercsptj|i||jj}dg|}|jj|_|jj|dd|jtd|_ |j dg |_ dS)N@@Fsequence_inputspeechbrain/soundchoice-g2pspn)r rrlexiconrupdate_from_iterableadd_unkrrNrOencode_sequence_torchrr)spn_token_encodedrrrrrrrrs    zFastSpeech2.__init__?cCs`t}t}t}|D]_}t} t} t} |} dd| D} || } tt| D].}| |}|D]}|sI| || d| dq4d| d<| |ddvrZd| d<q,|| || || q t}d}tt|D]}||} |j|  |j }t || |j } |jjd|d| d }t |d}tt||D]}|||dkr||qt}t|jdD]}|||||vr||jqt | |j }||||jdkr|jd}qvt t|| |j }|t|D]\}}|||dt|f<q|j||||d S) Computes mel-spectrogram for a list of texts Arguments --------- texts: List[str] texts to be converted to spectrogram pace: float pace for the speech synthesis pitch_rate : float scaling factor for phoneme pitches energy_rate : float scaling factor for phoneme energies Returns ------- tensors of output spectrograms, output lengths and alignments cSsg|]}|qSr)strip)r(wordrrrr+rz+FastSpeech2.encode_text..rr-z:;-,.!?rNpace pitch_rate energy_rate)listsplitrOr_risspaceappendrrrrZr$r% LongTensorrmodulesr unsqueezenonzeroreshapetolistshaper)rzero_ enumerater?)rr8rrrphoneme_labelslast_phonemes_combinedpunc_positionslabel phoneme_label last_phonemes punc_positionwordswords_phonemesrhwords_phonemes_seqphonemeall_tokens_with_spn max_seq_len token_seq spn_preds spn_to_addjtokens_with_spn token_idxtokens_with_spn_tensor_paddedseq_idxseqrrrrBs             zFastSpeech2.encode_textc Cg}d}|D]}|j||j}||jdkr |jd}||qtt |||j} | t |D]\} } | | | dt | f<q:|j | |||dSaComputes mel-spectrogram for a list of phoneme sequences Arguments --------- phonemes: List[List[str]] phonemes to be converted to spectrogram pace: float pace for the speech synthesis pitch_rate : float scaling factor for phoneme pitches energy_rate : float scaling factor for phoneme energies Returns ------- tensors of output spectrograms, output lengths and alignments rNr rrrrZr$rrr%rrrrr? rphonemesrrr all_tokensrrr tokens_paddedrrrrrencode_phoneme,   zFastSpeech2.encode_phonemec Csdt |jj||||d\}}}}}} }}|dd}Wdn1s'wY|||| fSaHBatch inference for a tensor of phoneme sequences Arguments --------- tokens_padded : torch.Tensor A sequence of encoded phonemes to be converted to spectrogram pace : float pace for the speech synthesis pitch_rate : float scaling factor for phoneme pitches energy_rate : float scaling factor for phoneme energies Returns ------- post_mel_outputs : torch.Tensor durations : torch.Tensor pitch : torch.Tensor energy : torch.Tensor rrr-Nr%r1rr transpose rrrrrr|post_mel_outputs durationspitchenergyrrrr?,s&    zFastSpeech2.encode_batchcC|j|g|||dSaBatch inference for a tensor of phoneme sequences Arguments --------- text : str A text to be converted to spectrogram pace : float pace for the speech synthesis pitch_rate : float scaling factor for phoneme pitches energy_rate : float scaling factor for phoneme energies Returns ------- Encoded text rrBrrArrrrrrrCY zFastSpeech2.forwardrrr) rDrErFrGrHrrBrr?rCrIrrrrr`s  l 1 -rcs\eZdZdZddgZfddZdddZd d Z dd d Z dd dZ dddZ Z S)FastSpeech2InternalAlignmenta A ready-to-use wrapper for Fastspeech2 with internal alignment(text -> mel_spec). Arguments --------- *args : tuple **kwargs : dict Arguments are forwarded to ``Pretrained`` parent class. Example ------- >>> tmpdir_tts = getfixture('tmpdir') / "tts" >>> fastspeech2 = FastSpeech2InternalAlignment.from_hparams(source="speechbrain/tts-fastspeech2-internal-alignment-ljspeech", savedir=tmpdir_tts) # doctest: +SKIP >>> mel_outputs, durations, pitch, energy = fastspeech2.encode_text(["Mary had a little lamb."]) # doctest: +SKIP >>> items = [ ... "A quick brown fox jumped over the lazy dog", ... "How much wood would a woodchuck chuck?", ... "Never odd or even" ... ] >>> mel_outputs, durations, pitch, energy = fastspeech2.encode_text(items) # doctest: +SKIP >>> # One can combine the TTS model with a vocoder (that generates the final waveform) >>> # Initialize the Vocoder (HiFIGAN) >>> tmpdir_vocoder = getfixture('tmpdir') / "vocoder" >>> from speechbrain.inference.vocoders import HIFIGAN >>> hifi_gan = HIFIGAN.from_hparams(source="speechbrain/tts-hifigan-ljspeech", savedir=tmpdir_vocoder) # doctest: +SKIP >>> # Running the TTS >>> mel_outputs, durations, pitch, energy = fastspeech2.encode_text(["Mary had a little lamb."]) # doctest: +SKIP >>> # Running Vocoder (spectrogram-to-waveform) >>> waveforms = hifi_gan.decode_batch(mel_outputs) # doctest: +SKIP r rcsXtj|i||jj}dg|}|jj|_|jj|dd|jtd|_ dS)NrFrr) r rrrrrrrrNrOrrrrrs   z%FastSpeech2InternalAlignment.__init__rc Cst}d}|D]$}||j|}|t|krt|}|j||j} | | qt t|||j} | t |D]\} } | | | dt| f<q@|j| |||dS)rrNr)r_g2p_keep_punctuationsrOrrrrrZr$rr%rrrr?) rr8rrrrrrphonemes_with_puncrrrrrrrrBs.   z(FastSpeech2InternalAlignment.encode_textcCsltd|}|D] }|dd}|dd}|dd}|dd}|||}qtd|}z||}WntyItd|tYnwd|d }g} d } z|D]} | d vrn| || d| d 7} qY| | qYWn1t yt d ||D]} | d vr| | } dd| D} | | q| | qYnwd| vr| dd| vs| S)zBdo grapheme to phoneme and keep the punctuations between the wordsz\w+[-':\.][-':\.\w]*\w+-':.z[\w]+|[-!'(),.:;? ]zerror with text: rRrz -!'(),.:;? r-zEDo g2p word by word because of unexpected outputs from g2p for text: cSsg|]}|dkr|qS)rRr)r(rhrrrr+szGFastSpeech2InternalAlignment._g2p_keep_punctuations..)refindallreplace RuntimeErrorloggerinfoquitr`rextendr IndexErrorwarningrOremove)r g2p_modelrA special_words special_wordrmpall_r word_phonemesrcountrhpp_without_spacerrrrsP                z3FastSpeech2InternalAlignment._g2p_keep_punctuationsc Crrrrrrrrrz+FastSpeech2InternalAlignment.encode_phonemec Cslt$|jj||||d\ }}}}}} }}}}}}|dd}Wdn1s+wY|||| fSrrrrrrr?2s.   z)FastSpeech2InternalAlignment.encode_batchcCrrrrrrrrCcrz$FastSpeech2InternalAlignment.forwardr) rDrErFrGrHrrBrrr?rCrIrrrrrps  43 1 1r)rGrrr%rUr2!speechbrain.inference.classifiersrspeechbrain.inference.encodersr speechbrain.inference.interfacesrspeechbrain.inference.textrspeechbrain.utils.fetchingrspeechbrain.utils.loggerr"speechbrain.utils.text_to_sequencerrDrr rJrrrrrrs*       \b