o i\@sdZddlmZddlmZddlmZmZmZddl Z ddl m Z ddl m Z ddlmZmZmZmZmZdd lmZmZmZmZdd lmZdd lmZdd lmZdd lm Z deiZ!eeeeedZ"ee j#edkrvddl$m%Z%nedddZ%Gddde Z&dS)zVITS module for GAN-TTS task.)contextmanager) LooseVersion)AnyDictOptionalN)check_argument_types) AbsGANTTS)HiFiGANMultiPeriodDiscriminatorHiFiGANMultiScaleDiscriminator)HiFiGANMultiScaleMultiPeriodDiscriminatorHiFiGANPeriodDiscriminatorHiFiGANScaleDiscriminator)DiscriminatorAdversarialLossFeatureMatchLossGeneratorAdversarialLossMelSpectrogramLoss) get_segments) VITSGenerator)KLDivergenceLoss)force_gatherablevits_generator)hifigan_period_discriminatorhifigan_scale_discriminator"hifigan_multi_period_discriminator!hifigan_multi_scale_discriminator.hifigan_multi_scale_multi_period_discriminatorz1.6.0)autocastTccs dVdS)NenabledrrM/home/ubuntu/.local/lib/python3.10/site-packages/espnet2/gan_tts/vits/vits.pyr/s rc#s eZdZdZddiddddddd dd d d d ddddddddddddddddddd d!d"d#id$d#d%d&d'dd(dd)d&d*d+d,gd-d.gd/d0gd1d2gd3gd3gd3gd4dd5d6d7d8d9dd:dd;d#ddd=d? d@ddAddddBddgdCdDdEd8dgdFdGdHd!iddIdJ dIgdKddd6d=gd gdLdEddGdHd!iddIdM dNdIdOdPdIdOdPdIdIddQddEdRddSdTdUdddV dWdXdYdWdWdfdZed[ed\ed]ed^eeefd_ed`eeefdaeeefdbeeefdceeefddeeefdeedfedgedhediedje f"fdkdl Z e dmdnZ e dodpZ    ddqejdrejdsejdtejduejdvejdweejdxeejdyeejdze d{eeeffd|d}Z   ddqejdrejdsejdtejduejdvejdweejdxeejdyeejd{eeeffd~dZ   ddqejdrejdsejdtejduejdvejdweejdxeejdyeejd{eeeffddZ      W  Iddqejdseejdweejdxeejdyeejdeejdedededeede d{eeejffddZZS)VITSaNVITS module (generator + discriminator). This is a module of VITS described in `Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech`_. .. _`Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech`: https://arxiv.org/abs/2006.04558 i"Vrhidden_channelsspksNlangs spk_embed_dimglobal_channels segment_size text_encoder_attention_headstext_encoder_ffn_expandtext_encoder_blocks$text_encoder_positionwise_layer_typeconv1d*text_encoder_positionwise_conv_kernel_size+text_encoder_positional_encoding_layer_typerel_pos&text_encoder_self_attention_layer_type rel_selfattntext_encoder_activation_typeswishtext_encoder_normalize_beforeTtext_encoder_dropout_rateg?$text_encoder_positional_dropout_rateg#text_encoder_attention_dropout_rate"text_encoder_conformer_kernel_size!use_macaron_style_in_text_encoder"use_conformer_conv_in_text_encoderdecoder_kernel_sizedecoder_channelsidecoder_upsample_scales)rFr,r,decoder_upsample_kernel_sizes)rHr.r.decoder_resblock_kernel_sizes)r@ decoder_resblock_dilations)r4rJuse_weight_norm_in_decoderposterior_encoder_kernel_sizerMposterior_encoder_layersrHposterior_encoder_stacksposterior_encoder_base_dilationposterior_encoder_dropout_rate$use_weight_norm_in_posterior_encoderrJg?) flow_flowsflow_kernel_sizeflow_base_dilation flow_layersflow_dropout_rateuse_weight_norm_in_flowuse_only_mean_in_flow)stochastic_duration_predictor_kernel_size*stochastic_duration_predictor_dropout_rate#stochastic_duration_predictor_flows-stochastic_duration_predictor_dds_conv_layersr AvgPool1d) kernel_sizestridepadding))rMrJi)r,r,r.r.r4 LeakyReLUnegative_slopeF) in_channels out_channels kernel_sizeschannelsmax_downsample_channels max_groupsbiasdownsample_scalesnonlinear_activationnonlinear_activation_paramsuse_weight_normuse_spectral_norm)r,rJrMr@rK)rJrJrJrJr4) rirjrkrlrprmrorqrrrsrt)scalesscale_downsample_poolingscale_downsample_pooling_paramsscale_discriminator_paramsfollow_official_normperiodsperiod_discriminator_paramsmse)average_by_discriminators loss_type)r}average_by_layersinclude_final_outputshannPr) fsn_fft hop_length win_lengthwindown_melsfminfmaxlog_base?gF@g@idimodim sampling_rategenerator_typegenerator_paramsdiscriminator_typediscriminator_paramsgenerator_adv_loss_paramsdiscriminator_adv_loss_paramsfeat_match_loss_paramsmel_loss_params lambda_adv lambda_mellambda_feat_match lambda_dur lambda_klcache_generator_outputscstsJtt|}|dkr|j||d|di||_t|}|di||_tdi||_ t di| |_ t di| |_ tdi| |_t|_| |_| |_||_||_||_||_d|_||_|jj|_|jj|_|jj|_dS)aInitialize VITS module. Args: idim (int): Input vocabrary size. odim (int): Acoustic feature dimension. The actual output channels will be 1 since VITS is the end-to-end text-to-wave model but for the compatibility odim is used to indicate the acoustic feature dimension. sampling_rate (int): Sampling rate, not used for the training but it will be referred in saving waveform during the inference. generator_type (str): Generator type. generator_params (Dict[str, Any]): Parameter dict for generator. discriminator_type (str): Discriminator type. discriminator_params (Dict[str, Any]): Parameter dict for discriminator. generator_adv_loss_params (Dict[str, Any]): Parameter dict for generator adversarial loss. discriminator_adv_loss_params (Dict[str, Any]): Parameter dict for discriminator adversarial loss. feat_match_loss_params (Dict[str, Any]): Parameter dict for feat match loss. mel_loss_params (Dict[str, Any]): Parameter dict for mel loss. lambda_adv (float): Loss scaling coefficient for adversarial loss. lambda_mel (float): Loss scaling coefficient for mel spectrogram loss. lambda_feat_match (float): Loss scaling coefficient for feat match loss. lambda_dur (float): Loss scaling coefficient for duration loss. lambda_kl (float): Loss scaling coefficient for KL divergence loss. cache_generator_outputs (bool): Whether to cache generator outputs. r)vocabs aux_channelsNr)rsuper__init__AVAILABLE_GENERATERSupdate generatorAVAILABLE_DISCRIMINATORS discriminatorrgenerator_adv_lossrdiscriminator_adv_lossrfeat_match_lossrmel_lossrkl_lossrrrrrr_cacherr$r%r&)selfrrrrrrrrrrrrrrrrrgenerator_classdiscriminator_class __class__rr r?sJ    z VITS.__init__cCdS)z)Return whether or not speech is required.Trrrrr require_raw_speech zVITS.require_raw_speechcCr)z*Return whether or not vocoder is required.Frrrrr require_vocoderrzVITS.require_vocodertext text_lengthsfeats feats_lengthsspeechspeech_lengthssidsspembslidsforward_generatorreturnc Cs<| r|j||||||||| d S|j||||||||| d S)aPerform generator forward. Args: text (Tensor): Text index tensor (B, T_text). text_lengths (Tensor): Text length tensor (B,). feats (Tensor): Feature tensor (B, T_feats, aux_channels). feats_lengths (Tensor): Feature length tensor (B,). speech (Tensor): Speech waveform tensor (B, T_wav). speech_lengths (Tensor): Speech length tensor (B,). sids (Optional[Tensor]): Speaker index tensor (B,) or (B, 1). spembs (Optional[Tensor]): Speaker embedding tensor (B, spk_embed_dim). lids (Optional[Tensor]): Language index tensor (B,) or (B, 1). forward_generator (bool): Whether to forward generator. Returns: Dict[str, Any]: - loss (Tensor): Loss scalar tensor. - stats (Dict[str, float]): Statistics to be monitored. - weight (Tensor): Weight tensor to summarize losses. - optim_idx (int): Optimizer index (0 for G and 1 for D). ) rrrrrrrrr)_forward_generator_forward_discrminator) rrrrrrrrrrrrrr forwards.# z VITS.forwardc " Cs|d} |dd}|d}d} |jr|jdur)d} |j||||||| d} n|j} |jr7|jr7| s7| |_| \} }}}}}}|\}}}}}}t|||jj|jj |jjd}| | }t | |}Wdn1srwYt dd L|| |}||||||}t |}||}|||}||j}||j}||j}||j}||j}|||||}Wdn1swYt||||||d } t|| | f|j\}} }!| s|jsd|_|| |!dd S) aPerform generator forward. Args: text (Tensor): Text index tensor (B, T_text). text_lengths (Tensor): Text length tensor (B,). feats (Tensor): Feature tensor (B, T_feats, aux_channels). feats_lengths (Tensor): Feature length tensor (B,). speech (Tensor): Speech waveform tensor (B, T_wav). speech_lengths (Tensor): Speech length tensor (B,). sids (Optional[Tensor]): Speaker index tensor (B,) or (B, 1). spembs (Optional[Tensor]): Speaker embedding tensor (B, spk_embed_dim). lids (Optional[Tensor]): Language index tensor (B,) or (B, 1). Returns: Dict[str, Any]: * loss (Tensor): Loss scalar tensor. * stats (Dict[str, float]): Statistics to be monitored. * weight (Tensor): Weight tensor to summarize losses. * optim_idx (int): Optimizer index (0 for G and 1 for D). rr4r,TNFrrrrrrrx start_idxsr)r)generator_lossgenerator_mel_lossgenerator_kl_lossgenerator_dur_lossrgenerator_feat_match_losslossstatsweight optim_idx)size transpose unsqueezerrrtrainingrupsample_factorr)rtorchno_gradrrrsumfloatrrrrrrrdictitemrdevice)"rrrrrrrrrr batch_size reuse_cacheouts speech_hat_dur_nll_rz_maskouts_z_pm_plogs_plogs_qspeech_p_hatprrdur_lossadv_lossrrrrrrr rRst "                 zVITS._forward_generatorc  CsB|d} |dd}|d}d} |jr|jdur)d} |j||||||| d} n|j} |jr4| s4| |_| ^} }}}}t|||jj|jj|jjd}| | }| |}t dd | ||\}}||}Wdn1stwYt |||d }t||| f|j\}}}| s|jsd|_|||dd S) aPerform discriminator forward. Args: text (Tensor): Text index tensor (B, T_text). text_lengths (Tensor): Text length tensor (B,). feats (Tensor): Feature tensor (B, T_feats, aux_channels). feats_lengths (Tensor): Feature length tensor (B,). speech (Tensor): Speech waveform tensor (B, T_wav). speech_lengths (Tensor): Speech length tensor (B,). sids (Optional[Tensor]): Speaker index tensor (B,) or (B, 1). spembs (Optional[Tensor]): Speaker embedding tensor (B, spk_embed_dim). lids (Optional[Tensor]): Language index tensor (B,) or (B, 1). Returns: Dict[str, Any]: * loss (Tensor): Loss scalar tensor. * stats (Dict[str, float]): Statistics to be monitored. * weight (Tensor): Weight tensor to summarize losses. * optim_idx (int): Optimizer index (0 for G and 1 for D). rr4r,TNFrrr)discriminator_lossdiscriminator_real_lossdiscriminator_fake_lossr)rrrrrrrrr)rdetachrrrrrrr)rrrrrrrrrrrrrrrrrrr real_loss fake_lossrrrrrr rsV "        zVITS._forward_discrminatorMbX?皙? durations noise_scalenoise_scale_duralphamax_lenuse_teacher_forcingc  Cs|d}tj|dgtj|jd} |dur|d}|dur$|d}|dur/|ddd}| r`|dus7J|ddd}tj|dgtj|jd} |jj|| || |||| | d \}}}n|jj|| ||||||| | d \}}}t |d|d|dd S) a$Run inference. Args: text (Tensor): Input text index tensor (T_text,). feats (Tensor): Feature tensor (T_feats, aux_channels). sids (Tensor): Speaker index tensor (1,). spembs (Optional[Tensor]): Speaker embedding tensor (spk_embed_dim,). lids (Tensor): Language index tensor (1,). durations (Tensor): Ground-truth duration tensor (T_text,). noise_scale (float): Noise scale value for flow. noise_scale_dur (float): Noise scale value for duration predictor. alpha (float): Alpha parameter to control the speed of generated speech. max_len (Optional[int]): Maximum length. use_teacher_forcing (bool): Whether to use teacher forcing. Returns: Dict[str, Tensor]: * wav (Tensor): Generated waveform tensor (T_wav,). * att_w (Tensor): Monotonic attention weight tensor (T_feats, T_text). * duration (Tensor): Predicted duration tensor (T_text,). Nr4)dtyperr(r,) rrrrrrrrr) rrrrrdurrrrrr)wavatt_wduration) rtensorrlongrviewrr inferencer)rrrrrrrrrrrrrrrrrrrr rsX%        zVITS.inference)NNNT)NNN) NNNNNrrrNF)__name__ __module__ __qualname____doc__intstrrrrboolrpropertyrrrTensorrrrrr __classcell__rrrr r!4sP      !"#0(  7 8 _ c g lwxyz{|N       D   v   `     r!)T)'r  contextlibrdistutils.versionrtypingrrrr typeguardrespnet2.gan_tts.abs_gan_ttsrespnet2.gan_tts.hifiganr r r r r espnet2.gan_tts.hifigan.lossrrrrespnet2.gan_tts.utilsrespnet2.gan_tts.vits.generatorrespnet2.gan_tts.vits.lossr espnet2.torch_utils.device_funcsrrr __version__torch.cuda.amprr!rrrr s4