o wiD@s<ddlZddlmZddlZddlmmZddlm Z ddl m Z ddl m Z mZmZddlmZmZddlmZddlmZdd lmZmZdd lmZmZmZdd lmZdd l m!Z!m"Z"dd l#m$Z$m%Z%ddl&m'Z'ddl(m)Z)m*Z*ddl+m,Z,m-Z-dZ.zddl/Z/Wn e0ydZ.YnwGdddeeZ1dS)N)Dict) instantiate) WandbLogger) DictConfig OmegaConf open_dict)DiscriminatorLoss GeneratorLoss)MultiResolutionSTFTLoss)Vocoder)MultiPeriodDiscriminatorMultiResolutionDiscriminator)get_batch_sizeget_num_workersplot_spectrogram_to_numpy) Exportable)PretrainedModelInfo typecheck) AudioSignalMelSpectrogramType) NeuralType)compute_max_stepsprepare_lr_scheduler)logging model_utilsTFcseZdZdZd:deddffdd Zdd Zed d Zd d Z e ddZ e de de ide deidd;ddZddZddZddZdd6d7Zd8d9Z Z!S)? UnivNetModelzeUnivNet model (https://arxiv.org/abs/2106.07889) that is used to generate audio from mel spectrogram.NcfgtrainerTrainercs>t|}t|}tj||dt|j|_t|jddd|_t|j |jj |jj d|_ t |j jd|vr9|jndd|_t|j jd|vrI|jndd|_t|_t|_|j jj|_dd |jD|_d d |jD|_d d |jD|_t|j|j|j|_|j|_|jjj|_d|_ d|_!|j"r|j"j#j$|_!d|_%dS) N)rrT)highfreq use_grads)n_mel_channels hop_lengthdebugF)r#cSg|]}|dqS)r.0resr%r%`/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/tts/models/univnet.py Cz)UnivNetModel.__init__..cSr$)r%r&r%r%r)r*Dr+cSr$)r%r&r%r%r)r*Er+)&r#convert_model_config_to_dict_configmaybe_update_config_versionsuper__init__r preprocessoraudio_to_melspec_precessortrg_melspec_fn generatornfiltn_window_strider discriminatormpdr#r mrdrdiscriminator_lossr generator_loss resolutionsstft_resolutions fft_sizes hop_sizes win_lengthsr mrstft_loss stft_lamb_cfg sample_rate stft_bias input_as_mel _train_dldatasetload_precomputed_melautomatic_optimization)selfrr __class__r%r)r1.s0         zUnivNetModel.__init__c Cs:t|jj|jj|jjt|jt|jj t |j|jj dS)N) max_epochsaccumulate_grad_batcheslimit_train_batches num_workers num_samples batch_size drop_last) rrDrOrrPrQrlenrHrIrrUrLr%r%r)_get_max_stepsRs zUnivNetModel._get_max_stepscCs<|dur |dur td|dur|S|dur||Std)Nz5Either use warmup_steps or warmup_ratio for schedulerz2Specify warmup_steps or warmup_ratio for scheduler) ValueError max_steps warmup_steps warmup_ratior%r%r)get_warmup_steps]szUnivNetModel.get_warmup_stepsc Cs6|jj}t|d|dd}t|d|dur%d|jvr%|jj}t||j d}t|t |j |j d}|dur|jdd}|dusQ|dkrU|}tj||dd|ddd }t|d||d<||d<|ddt|dt|||jd }t|||jd }||g||gfS||gS) NFschedT)paramsr[rr\r]rZ) optimizerscheduler_configtrain_dataloader)rDoptimcopyr set_structpopr_rr5 parameters itertoolschainr:r9getrXrr^rrH) rL optim_config sched_configoptim_goptim_dr[r\ scheduler_g scheduler_dr%r%r)configure_optimizersjsH         z!UnivNetModel.configure_optimizerscC |j|dSz^ Runs the generator, for inputs and outputs see input_types, and output_types xr5rLspecr%r%r)forwards zUnivNetModel.forwardry)BCTaudio)r{r}) input_types output_types torch.tensorreturncCs||ddS)Nryr,)squeezerxr%r%r)convert_spectrogram_to_audiosz)UnivNetModel.convert_spectrogram_to_audioc Cs|jr |\}}}n |\}}|||\}}|d}|j|d}||d|\}}|\} } | |j|| d\} } }}|j | | d\} }}|j || d\}}}}|j ||d\}}}|| }| || | |j|d|d|d\}}t|}t|}|||j}|j||d\}} }}|j ||d\}}}}|j| d\}}|j|d\}}|||}| || ||||||| |||j| jddd }|j|d d d |jd |d d d ddS)Nr,ru)yy_hat)disc_real_outputsdisc_generated_outputs)rvr input_lengths) disc_outputsrlr) g_loss_sc g_loss_mag g_loss_mrstftg_loss_gen_mpdg_loss_gen_mrdg_loss d_loss_mpd d_loss_mrdd_loss global_steprT)on_step sync_dist g_mrstft_lossF)prog_barloggerr)rGr3 unsqueezer5r4r optimizers zero_gradr9detachr;r:manual_backwardsteprBtorchstackmeanrCr<r param_groupslog_dictlog)rLbatch batch_idxr~ audio_len audio_mel_ audio_predaudio_pred_melrnrompd_score_real mpd_score_gen loss_disc_mpdmrd_score_real mrd_score_gen loss_disc_mrdloss_dloss_scloss_mag loss_mrstft loss_gen_mpd loss_gen_mrdloss_gmetricsr%r%r) training_steps\            zUnivNetModel.training_stepc Cs|jr|\}}}|jdg|jd}n |\}}|||\}}||d}|||d}|||\} } |jrA|||\} } ||d|\} } t|| }|jd|iddd|dkrPt|j t rRt rTg}g}t t d|jdD]}|tj||d||fjd||jd tj||dd||fjd d ||jd tj||d||fjd ||jd g7}|tjt||ddd||fjd |dtjt| |ddd||fjd|dtjt| |ddd||fjd|dg7}|jrC|tjt| |ddd||fjd|dg7}qy|j j||ddSdSdSdS)Nr,rrval_lossT)on_epochrz real audio )captionrEfloat32zgenerated audio zdenoised audio z input mel )rz output mel z denoised mel zgt mel )r~specs)rGshaper3 _bias_denoiserFl1_lossr isinstancerr HAVE_WANDBrangeminwandbAudiodatacpunumpyrEastypeImager experimentr)rLrrr~rr audio_mel_lenr pred_denoisedpred_denoised_melrgt_mel gt_mel_lenrloss_melclipsrir%r%r)validation_stepst   $&&&& zUnivNetModel.validation_stepc Csdd}dd}|jdus|jjd|jdkr@|tj||jdd}||\|_}|jdddddfdddddf|_||\}}||jdd |j}t|d }|||d } | S) NcSs`tj|dddddd}t|}|d|d}}t|d|d}t||}||fS) Nr,T)n_fftr" win_lengthreturn_complex).r).r,r-)rstftr view_as_realsqrtatan2)rvcomprealimagmagsphaser%r%r)r's   z(UnivNetModel._bias_denoise..stftcSsBtj|t||t|gdd}tjt|dddd}|S)N)dimrr)rr"r)rrcossinistftview_as_complex)rrrrvr%r%r)r/s&z)UnivNetModel._bias_denoise..istftr)devicerdenoise_strengthg{Gzd?gr,) rFrr zeros_likerrrkclampr) rLr~melrr audio_biasr audio_mags audio_phaseaudio_denoisedr%r%r)r&s ,  zUnivNetModel._bias_denoiseTtrainshuffle_should_benamecCsd|vs t|jtstd|d|vst|jts"td||rad|jvrPtd|d|dt|d d |j_Wdn1sJwYn#|jjs`t d |d |d n|ss|jjrst d |d |d t |j}t j j j|fd|ji|jS)NrIzNo dataset for dataloader_paramszNo dataloder_params for shufflez"Shuffle should be set to True for z's zE dataloader but was not found in its config. Manually setting to TrueTzThe z dataloader for z has shuffle set to False!!!z has shuffle set to True!!! collate_fn)rrIrrYrrwarningrrerrorrrutilsr DataLoaderr)rLrrrrIr%r%r)__setup_dataloader_from_configAs(    z+UnivNetModel.__setup_dataloader_from_configcCs|||_dSN)+_UnivNetModel__setup_dataloader_from_configrHrLrr%r%r)setup_training_dataVsz UnivNetModel.setup_training_datacCs|j|ddd|_dS)NF validation)rr)r_validation_dlrr%r%r)setup_validation_dataYsz"UnivNetModel.setup_validation_datacCsdSrr%rr%r%r)setup_test_data\szUnivNetModel.setup_test_dataOptional[Dict[str, str]]cCs<g}tddd|d}||tddd|d}|||S)Ntts_en_lj_univnetznhttps://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_lj_univnet/versions/1.7.0/files/tts_en_lj_univnet.nemozThis model is trained on LJSpeech sampled at 22050Hz, and has been tested on generating female English voices with an American accent.)pretrained_model_namelocation descriptionclass_tts_en_libritts_univnetzhttps://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_libritts_univnet/versions/1.7.0/files/tts_en_libritts_multispeaker_univnet.nemozThis model is trained on all LibriTTS training data (train-clean-100, train-clean-360, and train-other-500) sampled at 22050Hz, and has been tested on generating English voices.)rappend)clslist_of_modelsmodelr%r%r)list_available_models_s   z"UnivNetModel.list_available_modelscKs4|jdurz|jWdStyYdSwdSr)r5remove_weight_normrY)rLkwargsr%r%r)_prepare_for_exportus  z UnivNetModel._prepare_for_exportcCsdtdtiS)Nry)r{Dr})rrrWr%r%r)r|s zUnivNetModel.input_typescCsdtdt|jiS)Nr~)r{Sr})rrrErWr%r%r)rszUnivNetModel.output_typesr,rcCs:t|}tj||jdd|f|j|jd}d|ifS)zs Generates input examples for tracing etc. Returns: A tuple of input examples. r2r6)rdtypery)nextrhrrandnrrr)rL max_batchmax_dimparrr%r%r) input_examples $ zUnivNetModel.input_examplecCrsrtrwrxr%r%r)forward_for_exports zUnivNetModel.forward_for_exportr)ryrrr)Tr)rr )r,r)"__name__ __module__ __qualname____doc__rr1rX staticmethodr^rrrrzrrrrrrrboolstrrrr r  classmethodrrpropertyrrr$r% __classcell__r%r%rMr)r+s<$  1  ;B     r)2ritypingrrtorch.nn.functionalnn functionalr hydra.utilsrlightning.pytorch.loggers.wandbr omegaconfrrr*nemo.collections.tts.losses.hifigan_lossesrr &nemo.collections.tts.losses.stftlossesr nemo.collections.tts.models.baser ,nemo.collections.tts.modules.univnet_modulesr r (nemo.collections.tts.parts.utils.helpersrrr nemo.corernemo.core.classes.commonrrnemo.core.neural_types.elementsrr"nemo.core.neural_types.neural_typernemo.core.optim.lr_schedulerrr nemo.utilsrrrrModuleNotFoundErrorrr%r%r%r)s2