o }oil@sddlZddlmZmZddlmZddlmZddlm Z m Z m Z m Z m Z ddlZddlZddlZddlZddlmZddlmZmZmZddlmZdd lmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&dZ'zddl(Z(Wn e)ydZ'Ynwde ede efddZ*de e+de e+de+fddZ,eGdddZ-eGdddZ.eGdddZ/d ed!e+fd"d#Z0Gd$d%d%eZ1e&Gd&d'd'eZ2Gd(d)d)e1Z3Gd*d+d+e1Z4Gd,d-d-e1Z5dS).N)ABCabstractmethod) dataclass)Path)DictListOptionalTupleType) rearrange)CallbackLightningModuleTrainer)TensorBoardLogger)Logger) WandbLogger)Tensor) create_plot)logging) experimentalTFloggers logger_typecCsF|D]}t||rt|dr|jS|Sqtd|d|d)N experimentzCould not find z logger in .) isinstancehasattrr ValueError)rrloggerr^/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/tts/parts/utils/callbacks.py _get_logger+s   r model_namecheckpoint_pathtypecCs|du|duksJd|d|dt|}|dkr$ddlm}|}n|dkr1ddlm}|}ntd |d |durF||}| S|d rT||}| S||}| S) Nz?Must provide exactly one of vocoder model_name or checkpoint: (z, )hifiganr) HifiGanModelunivnet) UnivNetModelzUnknown vocoder type ''z.nemo) strnemo.collections.tts.modelsr&r(rfrom_pretrainedendswith restore_fromload_from_checkpointeval)r!r"r#r& model_typer(vocoderrrr _load_vocoder5s*     r3c@s0eZdZUeed<ejed<eed<eed<dS) AudioArtifactiddata sample_ratefilepathN) __name__ __module__ __qualname__r*__annotations__npndarrayintrrrrrr4Ps   r4c@s8eZdZUeed<ejed<eed<eed<eed<dS) ImageArtifactr5r6r8x_axisy_axisN)r9r:r;r*r<r=r>rrrrrr@Xs   r@c@s2eZdZUeed<eed<eed<dZeed<dS)LogAudioParams vocoder_type vocoder_namevocoder_checkpoint_pathF log_audio_gtaN)r9r:r;r*r<rGboolrrrrrCas rCr8returncCs t|d}|tjd}|S)N_)r* with_suffixreplaceossep)r8 path_prefixfile_idrrr create_idisrRc @s>eZdZe d dedededeee ee ffddZ dS) ArtifactGeneratorFmodel batch_dict initial_logrIcCsdS)a Create artifacts for the input model and test batch. Args: model: Model instance being trained to use for inference. batch_dict: Test batch to generate artifacts for. initial_log: Flag to denote if this is the initial log, can be used to save ground-truth data only once. Returns: List of audio and image artifacts to log. Nr)selfrTrUrVrrrgenerate_artifactspsz$ArtifactGenerator.generate_artifactsNF) r9r:r;rr rrHr rr4r@rXrrrrrSosrSc@seZdZdZ      d#deedejjj de ee de d e e d e ee d ed efd dZdede de fddZdede de fddZd$dedede e de fddZdedefdd Zdedefd!d"ZdS)%LoggingCallbacka Callback which can log artifacts (eg. model predictions, graphs) to local disk, Tensorboard, and/or WandB. Args: generators: List of generators to create and log artifacts from. data_loader: Data to log artifacts for. log_epochs: Optional list of specific training epoch numbers to log artifacts for. epoch_frequency: Frequency with which to log output_dir: Optional local directory. If provided, artifacts will be saved in output_dir. loggers: Optional list of loggers to use if logging to tensorboard or wandb. log_tensorboard: Whether to log artifacts to tensorboard. log_wandb: Whether to log artifacts to WandB. NF generators data_loader log_epochsepoch_frequency output_dirrlog_tensorboard log_wandbc Cs||_||_|r |ng|_||_|rt|nd|_|r|ng|_||_||_|r5t dt |jt |_ nt dd|_ |rRtsEtdt dt |jt|_nt dd|_t d|jjt d|jt d|jt d |jt d |jt d |jdS) NzCreating tensorboard loggerzNot using tensorbord loggerzWandb not installed.zCreating wandb loggerzNot using wandb loggerInitialized %s withz log_epochs: %sz epoch_frequency: %sz output_dir: %sz log_tensorboard: %sz log_wandb: %s)r\r]r^r_rr`rrarbrinfor rtensorboard_loggerdebug HAVE_WANDBrr wandb_logger __class__r9) rWr\r]r^r_r`rrarbrrr__init__s4     zLoggingCallback.__init__audiolog_dirstepcCs|r||j}|jjdddtj||j|jd|jr)|jj|j |j||jd|j rCt j |j|j|j df}|j |j |idSdS)NTparentsexist_ok)filer6 samplerate)tag snd_tensor global_stepr7)r7caption)r8parentmkdirsfwriter6r7re add_audior5rhwandbAudiolog)rWrkrlrmr8 wandb_audiorrr _log_audios zLoggingCallback._log_audioimagecCs|r||j}|jjdddnd}t||j|j|jd}|jr+|jj|j ||dd|j rBt j ||j df}|j |j |idSdS)NTrn)output_filepathr6rArBHWC)rs img_tensorru dataformats)rv)r8rwrxrr6rArBre add_imager5rhr|Imager~)rWrrlrmr8 image_plot wandb_imagerrr _log_images  zLoggingCallback._log_imager audio_list image_listrucCsN|dur |jddd|D] }|j|||dq |D] }|j|||dqdS)zLog audio and image artifacts.NTrn)rkrlrm)rrlrm)rxrr)rWrrrlrurkrrrr_log_artifactsszLoggingCallback._log_artifactstrainerrTc Csg}g}|jD]1}|D]\}}t|tjr||j||<q |jD]}|j||dd\} } || 7}|| 7}q#qt |t |krGdkrPnnt ddS|j rX|j dnd} |j ||| ddS)zLog initial data artifacts.T)rTrUrVrz,List are empty, no initial artifacts to log.Ninitialrrrl)r]itemsrtorchrtodevicer\rXlenrrfr`r) rWrrTrrrUkeyvalue generatorrkimagesrlrrr on_fit_starts"      zLoggingCallback.on_fit_startc Csd|j}||jvr||jdkrdSg}g}|jD]0}|D]\}}t|tjr2||j ||<q |j D]} | j ||d\} } || 7}|| 7}q6qt |t |krYdkrcnnt d|dS|jrn|jd|nd} |j||| ddS)z%Log artifacts at the end of an epoch.r[rN)rTrUz0List are empty, no artifacts to log at epoch %d.epoch_r) current_epochr^r_r]rrrrrrr\rXrrrfr`r) rWrrTepochrrrUrrrrkrrlrrron_train_epoch_ends(       z"LoggingCallback.on_train_epoch_end)Nr[NNFF)Nr)r9r:r;__doc__rrSrutilsr6 DataLoaderrr?rrrHrjr4rr@rlistrrr rrrrrrrZs<   +  rZc @s>eZdZdZ d dedededeee ee ffddZ d S) VocoderArtifactGeneratorz6 Generator for logging Vocoder model outputs. FrTrUrVrIcCsx|d}|d}dd|D}|d}|d}g} |rZtt||D]/\} \} } t| d| d} || d|| f}td | || |jd }| |q&| gfS| ||\}}t |j |d }t |d }Wdn1s|wYtt||D]/\} \} } t| d| d }|| d|| f}td| |||jd }| |q| gfS)N dataset_namesaudio_filepathscSg|]}t|qSrrR.0prrr +z?VocoderArtifactGenerator.generate_artifacts..rk audio_lens/_gt.wav audio_gt_r5r6r8r7specz B 1 T -> B T.wavaudio_)get enumerateziprcpunumpyr4r7appendaudio_to_melspec_precessorrno_gradforwardr )rWrTrUrVrr audio_idsrk audio_lenaudio_artifactsi dataset_nameaudio_id audio_gt_path audio_gt_iaudio_artifactrspec_len audio_predaudio_pred_path audio_pred_irrrrX%sD         z+VocoderArtifactGenerator.generate_artifactsNrY) r9r:r;rr rrHr rr4r@rXrrrrr src @seZdZdZddededefddZ dd ed eed eed e d e def ddZ d ed eed eed e d e f ddZ dd ede dede eeeeffddZdS)AudioCodecArtifactGeneratorz: Generator for logging Audio Codec model outputs. TF log_audio log_encodinglog_dequantizedcCsP||_||_||_td|jjtd|jtd|jtd|jdS)Nrcz log_audio: %sz log_encoding: %sz log_dequantized: %s)rrrrrfrir9)rWrrrrrrrjYsz$AudioCodecArtifactGenerator.__init__rTrrrkr save_inputcCs*|jsgSt|||d\}}Wdn1swYg} tt||D]/\} \} } t| d| d} || d|| f}td| || |j d}| |q*|rtt||D]/\} \} } t| d| d}|| d|| f}td| |||j d}| |qc| S) aGenerate audio artifacts. Args: model: callable model, outputs (audio_pred, audio_pred_len) dataset_names: list of dataset names for the examples in audio batch audio_ids: list of IDs for the examples in audio batch audio: tensor of input audio signals, shape (B, T) audio_len: tensor of lengths for each example in the batch, shape (B,) save_input: if True, save input audio signals rkrNrz_audio_out.wav audio_out_rz _audio_in.wav audio_in_) rrrrrrrrr4r7r)rWrTrrrkrrraudio_pred_lenrrrrrrr audio_in_path audio_in_irrr_generate_audiofs8   z+AudioCodecArtifactGenerator._generate_audiocCsg}|js |js |St|j||d\}}Wdn1s"wY|jrdtt||D]2\} \} } t| d| d} || ddd|| f } t d| | | ddd}| |q1|jsi|St|j ||d }|j ||d }Wdn1swYtt||D]2\} \} } t| d| d }|| ddd|| f }t d | ||ddd}| |q|S) aGenerate image artifacts. Args: model: model, needs to support `model.encode_audio`, `model.quantize` and `model.dequantize` dataset_names: list of dataset names for the examples in audio batch audio_ids: list of IDs for the examples in audio batch audio: tensor of input audio signals, shape (B, T) audio_len: tensor of lengths for each example in the batch, shape (B,) rNrz _encoded.pngencoded_ Audio FramesChannelsr5r6r8rArB)encoded encoded_len)tokens tokens_lenz_dequantized.png dequantized_)rrrr encode_audiorrrrrr@rquantize dequantize)rWrTrrrkrimage_artifactsrrrrr encoded_path encoded_iencoded_artifactr dequantizeddequantized_path dequantized_idequantized_artifactrrr_generate_imagessH  "  " z,AudioCodecArtifactGenerator._generate_imagesrUrVrIc Csh|d}|d}dd|D}|d}|d}|j||||||d} |j|||||d} | | fS) z Args: model: model used to process input to generate artifacts batch_dict: dictionary obtained form the dataloader initial_log: save input audio for the initial log rrcSrrrrrrrrrzBAudioCodecArtifactGenerator.generate_artifacts..rkr)rTrrrkrr)rTrrrkr)rrr) rWrTrUrVrrrrkrrrrrrrXs"    z.AudioCodecArtifactGenerator.generate_artifactsN)TFFrY)r9r:r;rrHrjr rr*rrrrr r4r@rXrrrrrTsL 6 9rc @seZdZdZ   ddededeefddZd ed e e d e e d e fd dZ de de defddZd ed e e d e e d e fddZd ed e e d e e d e fddZ dd ed e dedee ee effddZdS)FastPitchArtifactGeneratorac Generator for logging FastPitch model outputs. Args: log_spectrogram: Whether to log predicted spectrograms. log_alignment: Whether to log alignment graphs. audio_params: Optional parameters for saving predicted audio. Requires a vocoder model checkpoint for generating audio from predicted spectrograms. FNlog_spectrogram log_alignment audio_paramscCsN||_||_|sd|_d|_d|_dSd|_|j|_t|j|j|jd|_dS)NFT)r!r"r#) rrrrGr2r3rErFrD)rWrrrrrrrjs  z#FastPitchArtifactGenerator.__init__rTrrrUcCs|d}|d}|j||d\}}g} g} tt||D]\\} \} } t| d| d}|| d|| f}td| |||jjd}| |t| d| d}|| ddd|| f}t d | ||d d d }| |q| | fS) Nrkr input_signallengthrrrrz _spec_gt.pngspec_rrr) r preprocessorrrrrrr4 _sample_raterr@)rWrTrrrUrkrrrrrrrrrrr spec_gt_path spec_gt_i spec_artifactrrr_create_ground_truth_artifactss4   " z9FastPitchArtifactGenerator._create_ground_truth_artifactsmelsmels_len hop_lengthcCsh||jj}t|jj|d}Wdn1swY|}tj j ||d}||fS)Nr)r) rr2rrrconvert_spectrogram_to_audiorrlibrosacoreframes_to_samples)rWrrr voc_inputrmels_len_arrayaudio_pred_lensrrrr6s  z*FastPitchArtifactGenerator._generate_audiocCsvg}g}|d}|d}|dd} t|j||| d^} } } Wdn1s.wY|jrptt||D]2\} \}}t|d|d}| | ddd| | f }t d|||dd d }| |q=|j r|j | | |jjd \}}tt||D]0\} \}}t|d|d }|| d|| f }td ||||jjd}| |q||fS)Ntext text_lens speaker_id)r input_lensspeakerrz _spec.pngrrrrrrrrrr)rrrrrrrrrrr@rrrrrr4r2r7)rWrTrrrUrrrrr  mels_pred mels_pred_lenrKrrr spec_pathspec_irrrrrrrrr_generate_predictions?sN    "    z0FastPitchArtifactGenerator._generate_predictionsc Csg}g}|d}|d}|d} |d} |dd} |dd} |dd} |dd}|j||d \}}t!|j| | | | |||| d \ }}}}}}}}}}}}Wdn1sawY|jrt|d }tt||D]4\}\}}t |d |d }||d| |d||f }t d|||ddd}| |qu|jr|j|||jjd\}}tt||D]0\}\}}t |d |d}||d||f }td||||jjd}| |q||fS)Nrkrrralign_prior_matrixpitchenergyrr)rrrrr rmel_lens attn_priorz$B 1 T_spec T_text -> B T_text T_specrz _align.pngalign_rz Text Tokensrr z_gta.wav audio_gta_r)rrrrrrr rrrrrr@rrGrrr4r2r7) rWrTrrrUrrrkrrrrrrr rrr r rKattnrrr attn_pathattn_ialignment_artifactrrrrrrrr_generate_gta_predictionspsf           &    z4FastPitchArtifactGenerator._generate_gta_predictionsrVrIcCs|d}|d}dd|D}|r"|j||||d\}}||fSg} g} |js,|jr?|j||||d\} } | | 7} | | 7} |jsE|jrX|j||||d\} }| | 7} | |7} | | fS)NrrcSrrrrrrrrrzAFastPitchArtifactGenerator.generate_artifacts..)rTrrrU)rrrrrrGrr)rWrTrUrVrrraudio_gtspec_gtrrr spec_predaudio_gta_pred alignmentsrrrrXs.       z-FastPitchArtifactGenerator.generate_artifacts)FFNrY)r9r:r;rrHrrCrjr rr*rrrr?rrrr r4r@rXrrrrrsd   !  1 =r)6rNabcrr dataclassesrpathlibrtypingrrrr r rrr= soundfileryreinopsr lightning.pytorchr r rlightning.pytorch.loggersr lightning.pytorch.loggers.loggerrlightning.pytorch.loggers.wandbrr(nemo.collections.tts.parts.utils.helpersr nemo.utilsrnemo.utils.decoratorsrrgr|ModuleNotFoundErrorr r*r3r4r@rCrRrSrZrrrrrrrsR             4!