o }oi.B@snddlmZddlZddlmmZddlmZddl m Z ddl m Z ddl mZddlmZmZddlmZmZmZmZmZmZmZGd d d eZGd d d eZGd ddeZGdddeZGdddeZGdddeZGdddeZ GdddeZ!GdddeZ"GdddeZ#GdddeZ$Gdd d eZ%Gd!d"d"eZ&Gd#d$d$eZ'dS)%)ListN) rearrange)FilterbankFeatures)mask_sequence_tensor)get_mask_from_lengths)Loss typecheck) AudioSignal LengthsTypeLossType NeuralTypePredictionsTypeRegressionValuesTypeVoidTypecsJeZdZd deffdd ZeddZeddZed d Z Z S) MaskedLoss? loss_scalecstt|||_||_dSN)superr__init__rloss_fn)selfrr __class__`/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/tts/losses/audio_codec_loss.pyr%s zMaskedLoss.__init__cCs(tdttdtttdtdS)N)BDTr predictedtarget target_len)r r rtupler rrrr input_types*  zMaskedLoss.input_typescCdttdiSNloss elements_typer r r$rrr output_types2 zMaskedLoss.output_typescCsh|jd|jdks J|j||d}tj|dd}tj|ddtj|dd}t|}|j|}|S)N)inputr!dimr)min)shapertorchmeansumclampr)rr r!r"r)rrrforward8s  zMaskedLoss.forwardr) __name__ __module__ __qualname__floatrpropertyr%r-rr: __classcell__rrrrr$s  rc$eZdZddeffdd ZZS) MaskedMAELossrrc&tjjdd}tt|j||ddSNnone) reduction)rr)r6nnL1LossrrCrrrrrrrrKzMaskedMAELoss.__init__r;r<r=r>r?rrArrrrrCJrCcrB) MaskedMSELossrrcrDrE)r6rHMSELossrrNrrJrrrrQrKzMaskedMSELoss.__init__r;rLrrrrrNPrMrNcsBeZdZfddZeddZeddZeddZZ S) TimeDomainLosscstt|t|_dSr)rrPrrCrr$rrrrWs zTimeDomainLoss.__init__cC(tdttdtttdtdSN)rrr audio_real audio_gen audio_lenr r r#r r$rrrr%[r&zTimeDomainLoss.input_typescCr'r(r,r$rrrr-cr.zTimeDomainLoss.output_typescCs(t|d}t|d}|j|||d}|S)Nz B T -> B 1 T)r!r r")rr)rrTrUrVr)rrrr:is  zTimeDomainLoss.forward) r<r=r>rr@r%r-rr:rArrrrrPVs   rPc sbeZdZdZddedeedeedeffdd Zed d Z ed d Z e d dZ Z S)MultiResolutionMelLossa Multi-resolution log mel spectrogram loss. Args: sample_rate: Sample rate of audio. resolutions: List of resolutions, each being 3 integers ordered [num_fft, hop_length, window_length] mel_dims: Dimension of mel spectrogram to compute for each resolution. Should be same length as 'resolutions'. log_guard: Value to add to mel spectrogram to avoid taking log of 0. r sample_rate resolutionsmel_dims log_guardc stt|t|t|ksJt|_t|_tj |_ t ||D]\}\}}}t |||||ddd|dddddd} |j | q$dS)Nr1raddT)rYnfilt n_window_sizen_window_striden_fftpad_to mag_powerlog_zero_guard_typelog_zero_guard_valuemel_norm normalizepreemphdither use_grads)rrXrlenrC l1_loss_fnrN l2_loss_fnr6rH ModuleList mel_featuresziprappend) rrYrZr[r\mel_dimrbhop_lenwin_len mel_featurerrrr|s0 zMultiResolutionMelLoss.__init__cCrQrRrWr$rrrr%r&z"MultiResolutionMelLoss.input_typescCsttdttddS)Nr*)l1_lossl2_lossr,r$rrrr-s  z#MultiResolutionMelLoss.output_typesc Csd}d}|jD]&}|||d\}}|||d\} } ||j| ||d7}||j| ||d7}q|t|j}|t|j}||fS)Nr^)xseq_lenr)rprmrnrl) rrTrUrVrwrxrvmel_real mel_real_lenmel_gen_rrrr:s zMultiResolutionMelLoss.forwardr;)r<r=r>__doc__intrr?rr@r%r-rr:rArrrrrXqs(   rXcsbeZdZdZddeededeffdd Zd d Ze d d Z e d dZ e ddZ ZS)STFTLossaM Log magnitude STFT loss. Args: resolution: Resolution of spectrogram, a list of 3 numbers ordered [num_fft, hop_length, window_length] log_guard: Value to add to magnitude spectrogram to avoid taking log of 0. sqrt_guard: Value to add to when computing absolute value of STFT to avoid NaN loss. rh㈵> resolutionr\ sqrt_guardcsNtt|t|_|\|_|_|_|dt j |jdd||_ ||_ dS)NwindowF)periodic) rrrrCrrb hop_length win_lengthregister_bufferr6 hann_windowr\r)rrr\rrrrrs  zSTFTLoss.__init__cCsbtj||j|j|j|jdd}t|}t|d d|j }t ||j }t ||}|S)NT)rbrrrreturn_complexr/)r6stftrbrrr view_as_realsqrtpowr8rlogr\r)raudiospec_lenspecspec_magspec_logrrr_compute_spectrograms  zSTFTLoss._compute_spectrogramcCrQrRrWr$rrrr%r&zSTFTLoss.input_typescCr'r(r,r$rrrr-zSTFTLoss.output_typescCs>||jd}|j||d}|j||d}|j|||d}|S)Nr1)rrr)rrr)rrTrUrVr spec_realspec_genr)rrrr:s zSTFTLoss.forwardrr)r<r=r>rrrr?rrr@r%r-rr:rArrrrrs   rcsZeZdZdZddeededeffdd Zed d Zed d Z e d dZ Z S)MultiResolutionSTFTLossa[ Multi-resolution log magnitude STFT loss. Args: resolutions: List of resolutions, each being 3 integers ordered [num_fft, hop_length, window_length] log_guard: Value to add to magnitude spectrogram to avoid taking log of 0. sqrt_guard: Value to add to when computing absolute value of STFT to avoid NaN loss. rrrZr\rcs0tt|tjfdd|D|_dS)Ncsg|] }t|dqS))rr\r)r).0rr\rrr sz4MultiResolutionSTFTLoss.__init__..)rrrr6rHroloss_fns)rrZr\rrrrrs z MultiResolutionSTFTLoss.__init__cCrQrRrWr$rrrr%r&z#MultiResolutionSTFTLoss.input_typescCr'r(r,r$rrrr-rz$MultiResolutionSTFTLoss.output_typescCs4d}|jD] }|||||d7}q|t|j}|S)Nr^rS)rrl)rrTrUrVr)rrrrr: s  zMultiResolutionSTFTLoss.forwardr) r<r=r>rrr?rr@r%r-rr:rArrrrrs   rcsNeZdZdZd deffdd ZeddZedd Ze d d Z Z S) SISDRLossz SI-SDR loss based off of torchmetrics.functional.audio.sdr.scale_invariant_signal_distortion_ratio with added support for masking. :0yE>epsilonctt|||_dSr)rrrr)rrrrrr zSISDRLoss.__init__cCrQrRrWr$rrrr%r&zSISDRLoss.input_typescCr'r(r,r$rrrr-&rzSISDRLoss.output_typescCst||d}t|d}tj|ddd|}tj|ddd|}||}||}||}||}tj||ddd} tj|dddd} | |j| |j} | |} | |} tj| ddd}tj| ddd}||j||j}dt|}t| }|S) N)rylengthszB -> B 1rT)r3keepdimr/r2 )rrr6r8rlog10r7)rrTrUrVmask target_mean pred_meanr!predref_pred ref_targetalpha target_scaled distortiontarget_scaled_powerdistortion_powerratiosi_sdrr)rrrr:*s&   zSISDRLoss.forward)r) r<r=r>rr?rr@r%r-rr:rArrrrrs  rcsFeZdZdZfddZeddZeddZedd Z Z S) FeatureMatchingLossz Standard feature matching loss measuring the difference in the internal discriminator layer outputs (usually leaky relu activations) between real and generated audio, scaled down by the total number of discriminators and layers. cstt|dSr)rrrr$rrrrVszFeatureMatchingLoss.__init__cC"ttdggttdggdSNr*) fmaps_real fmaps_genr rr$rrrr%YzFeatureMatchingLoss.input_typescCr'r(r,r$rrrr-`r.z FeatureMatchingLoss.output_typesc Csdd}t||D]"\}}t||D]\}}t||}t|t|} || 7}qq|t|}|S)Nr^)rqr6absr7rl) rrrr) fmap_realfmap_gen feat_realfeat_gendiff feat_lossrrrr:fs  zFeatureMatchingLoss.forward r<r=r>rrr@r%r-rr:rArrrrrOs   rcsHeZdZdZd fdd ZeddZeddZed d Z Z S) RelativeFeatureMatchingLossa Relative feature matching loss as described in https://arxiv.org/pdf/2210.13438.pdf. This is similar to standard feature matching loss, but it scales the loss by the absolute value of each feature averaged across time. This might be slightly different from the paper which says the "mean is computed over all dimensions", which could imply taking the average across both time and features. Args: div_guard: Value to add when dividing by mean to avoid large/NaN values. MbP?crr)rrr div_guard)rrrrrrrz$RelativeFeatureMatchingLoss.__init__cCrrrr$rrrr%rz'RelativeFeatureMatchingLoss.input_typescCr'r(r,r$rrrr-r.z(RelativeFeatureMatchingLoss.output_typesc Csd}t||D]8\}}t||D].\}}tjt|dd}tjt||dd} | ||j} t| t|} || 7}qq|t|}|S)Nr^rr2)rqr6r7rrrl) rrrr)rrrr feat_meanrrrrrr:s  z#RelativeFeatureMatchingLoss.forward)rrrrrrrvs   rc@2eZdZeddZeddZeddZdS)GeneratorHingedLosscCdtdtgiSNdisc_scores_genrCrrr$rrrr%zGeneratorHingedLoss.input_typescCr'r(r,r$rrrr-rz GeneratorHingedLoss.output_typescCs6d}|D]}|ttd|7}q|t|}|S)Nr^r1)r6r7Frelurlrrr)disc_score_genrrrr:s  zGeneratorHingedLoss.forwardNr<r=r>r@r%r-rr:rrrrr  rc@r)GeneratorSquaredLosscCrrrr$rrrr%rz GeneratorSquaredLoss.input_typescCr'r(r,r$rrrr-rz!GeneratorSquaredLoss.output_typescCs4d}|D] }|td|d7}q|t|}|SNr^r1r/)r6r7rlrrrrr:s  zGeneratorSquaredLoss.forwardNrrrrrrrrc@r)DiscriminatorHingedLosscCtdtgtdtgdSNr)disc_scores_realrrr$rrrr%  z#DiscriminatorHingedLoss.input_typescCr'r(r,r$rrrr-rz$DiscriminatorHingedLoss.output_typescCs`d}t||D] \}}ttd|}ttd|}|||d7}q|t|}|Sr)rqr6r7rrrlrrrr)disc_score_realr loss_realloss_genrrrr:s zDiscriminatorHingedLoss.forwardNrrrrrr  rc@r)DiscriminatorSquaredLosscCrrrr$rrrr%rz$DiscriminatorSquaredLoss.input_typescCr'r(r,r$rrrr-rz%DiscriminatorSquaredLoss.output_typescCsXd}t||D]\}}td|d}t|d}|||d7}q|t|}|Sr)rqr6r7rlrrrrr:s z DiscriminatorSquaredLoss.forwardNrrrrrrrr)(typingrr6torch.nn.functionalrH functionalreinopsr1nemo.collections.asr.parts.preprocessing.featuresr#nemo.collections.common.parts.utilsr(nemo.collections.tts.parts.utils.helpersrnemo.core.classesrrnemo.core.neural_typesr r r r r rrrrCrNrPrXrrrrrrrrrrrrrs.     $ &E9%;'1