o }oiS@sddlZddlZddlmZmZddlmZmZmZm Z ddl Z ddl Z ddl mZddlmmZddlmZddlmZddlmZddlmZmZmZmZddlmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)dd l*m+Z+zddl,Z,d Z-Wn e.ydZ-Ynwzddl/Z/d Z0Wn e.ydZ0Ynwdbde1de1de1fddZ2de e1e1fde e1e1fde e1e1ffddZ3de1de1de1fddZ4de1de1de e1e1ffddZ5Gddde Z6Gddde Z7d d!Z8dcd"e9d#e9fd$d%Z:Gd&d'd'e Z;Gd(d)d)e ZGd.d/d/ej?Z@Gd0d1d1e ZAGd2d3d3e ZBGd4d5d5e ZCGd6d7d7e ZDGd8d9d9e ZEGd:d;d;e ZFGdd?d?e ZHGd@dAdAe ZIGdBdCdCe ZJGdDdEdEe ZKGdFdGdGe eZLGdHdIdIeLZMGdJdKdKeLZNGdLdMdMe ZOGdNdOdOe ZPGdPdQdQe ZQGdRdSdSe ZRGdTdUdUe ZSGdVdWdWe ZTGdXdYdYe ZUGdZd[d[e ZVGd\d]d]e ZWGd^d_d_e ZXGd`dadae ZYdS)dN)ABCabstractmethod)IterableListOptionalTuple) rearrange) AutoModel)!AudioToMelSpectrogramPreprocessor)ClampActivation HalfSnakeSnakemask_sequence_tensor) typecheck) NeuralModule) AudioSignalEncodedRepresentationIndex LengthsTypeMelSpectrogramTypeVoidType) NeuralType)loggingTF kernel_sizedilationreturncCs|||dS)Nrrrrd/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/tts/modules/audio_codec_modules.py get_padding9r!cCs(t|d|dt|d|df}|S)Nrr)r!)rrpaddingsrrr get_padding_2d=s$r$stridecCs||ddS)Nrrrrr%rrr get_down_sample_paddingBr"r'cCs$||d}||dd}||fS)Nrrr)rr%output_paddingpaddingrrr get_up_sample_paddingFs r*cs$eZdZfddZddZZS)SSLModelctt||_dSN)super__init__r from_pretrained ssl_model)selfslm_model_name __class__rr r/M zSSLModel.__init__cOs|j|i|Sr-)r1)r2argskwargsrrr forwardQr"zSSLModel.forward__name__ __module__ __qualname__r/r9 __classcell__rrr4r r+Ls r+cs^eZdZdZ       dfd d Zd d Zed dZeddZe ddZ Z S)SLMDiscriminatora'SLM Discriminator, as described in both the StyleTTS2 and Low Frame-Rate Speech Codec papers. Args: slm_model_name: Hugging Face Speech Language Models name. slm_sr: Speech Language Models input sampling rate. input_sr: Audio input sampling rate. slm_hidden: Speech Language Model hidden dim. slm_layers: Speech Language Model number of layers. initial_channel: discriminative head number of channels. use_spectral_norm: If True uses spectral normalization otherwise uses weight norm. microsoft/wavlm-base-plus>"V @Fc sttrtj|||_nd|_t||_|j |dkr't j j j jnt j j j}|t j|||dddd|_t |t j||dddd|t j|d|dddd|t j|d|dddddg|_|t j|ddd ddd|_dS) NFrr)r)r)rr))r.r/HAVE_TORCHAUDIO torchaudio transformsResampleresampler+ slm_modelfreezetorchnnutilsparametrizations weight_norm spectral_normConv1dpre ModuleListconvs conv_post) r2r3slm_srinput_sr slm_hidden slm_layersinitial_channeluse_spectral_normnorm_fr4rr r/cs   "zSLMDiscriminator.__init__cCs|j||ddj}tj|ddddjddd}||}g}|jD]}||}t |d }| | dq%| |}t|dd}||fS) NT) input_valuesoutput_hidden_statesrdimr) start_dimend_dim皙?)rNrM hidden_statesrPstack transposeflattenrWrYF leaky_reluappend unsqueezerZ)r2xfmaplayerrrr _forwards     zSLMDiscriminator._forwardcCtdttdtdSNBT_audio audio_real audio_genrrr2rrr input_types  zSLMDiscriminator.input_typescC:tdtgtdtgtdtggtdtggdSNrzCT_outrzDT_layerr scores_real scores_gen fmaps_real fmaps_genrrrrrr output_types  zSLMDiscriminator.output_typescCs<||\}}||\}}|dg|dg|g|gfSNr)rvrr)r2r}r~y_d_rfmap_ry_d_gfmap_grrr r9s zSLMDiscriminator.forward)r@rArBrCrDrEF) r;r<r=__doc__r/rvpropertyrrrr9r>rrr4r r?Us"%  r?cCs6||jdddt|jdddd}|S)z8 Normalized to have zero mean and unit variance rrdrfgHz>)meanrrrPsqrtvar)rbnormed_input_valuesrrr zero_mean_unit_var_normsrpath map_locationcKstj|p tj|}|rtj|fd|i|Str>t|d}tj|fd|i|WdS1s7wYdSt dt d)aLike torch.load but can load from other locations (e.g. s3:// , gs://). Args: path: Any path or url supported by fsspec. map_location: torch.device or str. cache: If True, cache a remote file locally for subsequent calls. It is cached under `get_user_data_dir()/tts_cache`. Defaults to True. **kwargs: Keyword arguments forwarded to torch.load. Returns: Object stored in path. rrbNzDCould not import fsspec. Loading a checkpoint link is not supported!zJfsspec is not installed but is necessary to download remote checkpoints !!) osrisdirisfilerPload HAVE_FSSPECfsspecopenrerrorModuleNotFoundError)rrr8is_localfrrr load_fsspecs $ rc&eZdZdfdd ZddZZS) PreEmphasis ףp= ?cs:t||_|dt|j dgdddS)Nfilter?r)r.r/ coefficientregister_bufferrP FloatTensorrr)r2rr4rr r/s *zPreEmphasis.__init__cCsDt|dks Jtjj|ddd}tjj||j dS)Nrrrrreflect) lensizerPrQ functionalpadrrconv1drsqueezer2rsrrr r9szPreEmphasis.forward)rr:rrr4r rsrcr)SELayercsTtt|td|_tt|||tjddt|||t |_ dS)NrTinplace) r.rr/rQAdaptiveAvgPool2davg_pool SequentialLinearReLUSigmoidfc)r2channel reductionr4rr r/s   zSELayer.__init__cCs@|\}}}}||||}||||dd}||Sr)rrviewr)r2rsbc_yrrr r9szSELayer.forward)rr:rrr4r rs rcs*eZdZdZdfdd ZddZZS) SEBasicBlockrNrcs~tt|tj||d|ddd|_t||_tj||dddd|_t||_ tj dd|_ t |||_ ||_||_dS)NrHrF)rr%r)bias)rr)rTr)r.rr/rQConv2dconv1 BatchNorm2dbn1conv2bn2rrelurse downsampler%)r2inplanesplanesr%rrr4rr r/s    zSEBasicBlock.__init__cCsj|}||}||}||}||}||}||}|jdur*||}||7}||}|Sr-)rrrrrrr)r2rsresidualoutrrr r9s         zSEBasicBlock.forward)rNr)r;r<r= expansionr/r9r>rrr4r rs rcseZdZdZddgdgddddiddd d d d d dddddddddddddddddddddd dd!dd"d#d$ddd%dd&dddd'd( ffd)d* Zd+d,Zd9d.d/Zd0d1Zd:d2d3Zd4d5Z d;d6e fd7d8Z Z S)<ResNetSpeakerEncoderzImplementation of the model H/ASP without batch normalization in speaker embedding. This model was proposed in: https://arxiv.org/abs/2009.14153 Adapted from: https://github.com/clovaai/voxceleb_trainer rE)rHrGrH) rEASPTfft_size win_lengthi hop_lengthframe_shift_msNframe_length_ms stft_pad_moder sample_raterArMF preemphasisr ref_level_db do_sound_normdo_trim_silencetrim_db<powerg?griffin_lim_itersnum_melsmel_fming@@ig@g;) mel_fmax spec_gain signal_norm min_level_dbsymmetric_normmax_norm clip_norm stats_path do_rms_normdb_levelc stt|||_||_||_||_||_||_t j d|ddddd|_ t j dd|_ t |d|_|d|_|t|d|d|_|jt|d|ddd|_|jt|d |d dd|_|jt|d|ddd|_t ||_|jrtr|||_nd|_t|jd } t t j|d| d dd t t d t jd |d| dd t j d d |_!|jdkr|d| } n|jdkr|d| d } nt"dt #| ||_$|%dS)NrrrH)rr%r)Tr)rr)r%rrrrrdSAPrzUndefined encoder)&r.rr/ encoder_type input_dim log_inputuse_torch_spec audio_configproj_dimrQrrrrrrr create_layerrlayer1layer2layer3layer4InstanceNorm1d instancenormrIget_torch_mel_spectrogram_class torch_specintrrV BatchNorm1dSoftmax attention ValueErrorrr _init_layers) r2r r layers num_filtersrr r r  outmap_sizeout_dimr4rr r/sD'       zResNetSpeakerEncoder.__init__cCs`|D])}t|tjrtjj|jdddqt|tjr-tj|jdtj|j dqdS)Nfan_outr)mode nonlinearityrr) modules isinstancerQrinitkaiming_normal_weightr constant_r)r2mrrr rrs   z!ResNetSpeakerEncoder._init_layersrc Csd}|dks|j||jkr&ttj|j||jd|ddt||j}g}|||j|||||j|_td|D] }|||j|q>tj|S)NrF)rr%r)rrrQrrrrqrange)r2blockrblocksr%rrrrrr rzs  z!ResNetSpeakerEncoder.create_layercGs ttj|}tj||Sr-)rQ ParameterrPrr&xavier_normal_)r2rrrrr new_parameters z"ResNetSpeakerEncoder.new_parametercCsZ|d|jr ||}|jr|d}||d}||}||}| |}| |}| |}| |}| |}||dd|d}||}|jdkrdtj||dd}n,|jdkrtj||dd}ttj|d|dd|djd d }t||fd}||dd}||}|rtjjj|ddd }|S) a{Forward pass of the model. Args: x (Tensor): Raw waveform signal or spectrogram frames. If input is a waveform, `torch_spec` must be `True` to compute the spectrogram on-the-fly. l2_norm (bool): Whether to L2-normalize the outputs. Shapes: - x: :math:`(N, 1, T_{in})` or :math:`(N, D_{spec}, T_{in})` rgư>rrfrrrdrgh㈵>)min)pre)squeeze_r rr logrrrrrrrrrrreshaperrrrPsumrclampcatrrrQr normalize)r2rsl2_normwmusgrrr r9s4             , zResNetSpeakerEncoder.forwardc Cs@tjt|dtjj|d|d|d|dtj|ddS)Nrrrrrr)rn_fftrr window_fnn_mels)rPrQrrrJrKMelSpectrogramhamming_window)r2r rrr rs z4ResNetSpeakerEncoder.get_torch_mel_spectrogram_classcheckpoint_pathcCs(t|tdd}|j|d|ddS)Ncpu)rmodel)strict)rrPdeviceload_state_dict)r2rCrFstaterrr load_checkpointsz$ResNetSpeakerEncoder.load_checkpointr)F)T) r;r<r=rr/rrr0r9rstrrJr>rrr4r rs|     V  / rcs4eZdZdZd dedeffdd Zdd ZZS) CodecActivationz Choose between activation based on the input parameter. Args: activation: Name of activation to use. Valid options are "elu" (default), "lrelu", and "snake". channels: Input dimension. elur activationchannelscszt|}|dkrt|_dS|dkr tj|_dS|dkr+t||_dS|dkr6t ||_dSt d|)NrNlrelusnake half_snakezUnknown activation ) r.r/lowerrQELUrOrP LeakyReLUr r r)r2rOrPr4rr r/s zCodecActivation.__init__cC ||Sr-)rOrrrr r9s zCodecActivation.forward)rNr) r;r<r=rrLrr/r9r>rrr4r rMsrMc s\eZdZdZ    ddedededed ed ef fd d Zd dZddZddZZ S)CausalConvTranspose1dNormz1ConvTranspose1d causal padding and normalization.rNT in_channels out_channelsrr%groupstrim_right_ratioc st||_|dur|n|}tj||||||d|_|jjd}|jjd}||}t ||j|_ ||j |_ tj j |j|_dS)N)r[rr)r.r/r\rQConvTranspose1dconvrr%mathceil padding_right padding_leftrRrSrT) r2rYrZrr%r[r\r padding_totalr4rr r/s    z"CausalConvTranspose1dNorm.__init__cCs0tjjj}ttjjdrtjjj}||jdS)NrT)rQrRrSrThasattrr^)r2rTrrr apply_weight_norm s  z+CausalConvTranspose1dNorm.apply_weight_normcCtj|jdSr-rQrRremove_weight_normr^rrrr rhz,CausalConvTranspose1dNorm.remove_weight_normcCs:||}|jd|j}|d|j|f}t||}|S)Nrf.)r^shaperarbr)r2inputs input_lenrkendrrr r9s  z!CausalConvTranspose1dNorm.forward)rNrT) r;r<r=rrr/rerhr9r>rrr4r rXs,rXcseZdZdZ      d dededed ed ed ed ed edeffdd ZddZde j de j fddZ e d!de j de eefdedefddZddZZS)"CausalConv1dNormz-Conv1d with causal padding and normalization.rzerosconstantTrYrZrr%rr[pad_modeextra_pad_moderc st||_|dkr|dkrtd|d|d|dtj||||||| |d|_|jjd}tj |jj dtj d}|jj d}tj |d|dtj d}|j d |d d |j d |d d |j d tj ||tj dd d tjj|j|_dS)NrzTCausalConv1dNorm has been initialized with stride > 1 and dilation > 1 (kernel_size=z stride=z , dilation=z).)rr[r padding_moderdtyper%F) persistentrrc)r.r/rrprintrQrVr^rrPtensorr%int64rrrRrSrT) r2rYrZrr%rr[rqrrrr4rr r/%s<    zCausalConv1dNorm.__init__cCrfr-rgrrrr rhTriz#CausalConv1dNorm.remove_weight_normrkrcCsX|jd}||j|j|jd}t|tjd}||j|j|j}||S)zSee `pad_for_conv1d`.rfr)rjrrcr%rPr`tory)r2rklengthn_frames ideal_lengthrrr _get_extra_padding_for_conv1dXs z.CausalConv1dNorm._get_extra_padding_for_conv1dzerorr#r"valuec Cs|jd}|\}}|dkstj||||St||}d}||kr0||d}tj|d|f}tj||||} | jd|} | dd| fS)zTiny wrapper around torch.nn.functional.pad, just to allow for reflect padding on small input. If this is the case, we insert extra 0 padding to the right before the reflection happens. rfrrr.N)rjrQrrmax) rkr#r"rr{rbramax_pad extra_padpaddedrmrrr _pad1dds   zCausalConv1dNorm._pad1dcCs:||}|j||j|f|jd}||}t||}|S)N)r")r~rrcrrr^r)r2rkrl extra_paddingrkrrr r9xs   zCausalConv1dNorm.forward)rrrrorpT)rr)r;r<r=rrrLboolr/rhrPTensorr~ staticmethodrfloatrr9r>rrr4r rn"sH  /  (rncsveZdZ    ddededededed eed effd d Zed dZeddZ ddZ e ddZ Z S) Conv1dNormrNrrYrZrr%rr)rqc sFt|s t||d}tj|||||||d}tjj||_dS)Nr)rYrZrr%r)rrs) r.r/r!rQrVrRrSrTr^) r2rYrZrr%rr)rqr^r4rr r/s   zConv1dNorm.__init__cCtdtttdtdSNrzrTrzrkrlrrtuplerrrrr r zConv1dNorm.input_typescCdtdtiSNrrrrrrr r zConv1dNorm.output_typescCrfr-rgrrrr rhrizConv1dNorm.remove_weight_normcC||}t||}|Sr-r^rr2rkrlrrrr r9  zConv1dNorm.forward)rrNr)r;r<r=rrrLr/rrrrhrr9r>rrr4r rs6  rc sbeZdZddededededef fdd Zed d Zed d Zd dZe ddZ Z S)ConvTranspose1dNormrrYrZrr%r[c sFtt||\}}tj||||||d|d}tjj||_dS)Nro)rYrZrr%r)r(rsr[) r.r/r*rQr]rRrSrTr^) r2rYrZrr%r[r)r(r^r4rr r/s  zConvTranspose1dNorm.__init__cCrrrrrrr rrzConvTranspose1dNorm.input_typescCrrrrrrr rrz ConvTranspose1dNorm.output_typescCrfr-rgrrrr rhriz&ConvTranspose1dNorm.remove_weight_normcCrr-rrrrr r9rzConvTranspose1dNorm.forwardrr) r;r<r=rr/rrrrhrr9r>rrr4r rs$  rc s~eZdZ  ddededeeefdeeefdeeeff fdd Zed d Zed d Zd dZ e ddZ Z S) Conv2dNormrrYrZrr%rc sTtt|t|ksJt||}tj||||||dd}tjj||_ dS)Nr)rYrZrr%rr)rs) r.r/rr$rQrrRrSrTr^)r2rYrZrr%rr)r^r4rr r/s   zConv2dNorm.__init__cCr)NrkrzrHrrrrrr rrzConv2dNorm.input_typescCr)Nrrrrrrr rrzConv2dNorm.output_typescCrfr-rgrrrr rhrizConv2dNorm.remove_weight_normcCrWr-)r^)r2rkrrr r9s zConv2dNorm.forward)rr) r;r<r=rrr/rrrrhrr9r>rrr4r rs*     rcsHeZdZdZd fdd ZeddZeddZed d Z Z S) PeriodDiscriminatora Period discriminator introduced in HiFi-GAN https://arxiv.org/abs/2010.05646 which attempts to discriminate phase information by looking at equally spaced audio samples. Args: period: Spacing between audio sample inputs. lrelu_slope: Slope to use for activation. Leaky relu with slope of 0.1 or 0.2 is recommended for the stability of the feature matching loss. rjc st||_t||_ttdddddtdddddtdddddtdddddtdddd dg|_tdddd |_ dS) Nrr)rFr)rHrr&rrirr) r.r/periodrQrVrOrXr conv_layersrZ)r2r lrelu_sloper4rr r/s   zPeriodDiscriminator.__init__cCrNaudioryrrrrr r!rzPeriodDiscriminator.input_typescCtdttdtgdS)Nrr)scorertrrrrr r'  z PeriodDiscriminator.output_typesc Cs|j\}}t|d}||jdkr&|j||j}t|d|fd}||}||d||j|j}g}|jD]}||d}||}||q7|j |d}||t|d}||fS)N B T -> B 1 TrrrrkB 1 T C -> B C T) rjrrrorrrrOrqrZ) r2r batch_sizetimern_padrtr^rrrr r9.s         zPeriodDiscriminator.forward)rj) r;r<r=rr/rrrrr9r>rrr4r rs   rcsReZdZdZd deeffdd ZeddZed d Z e d d Z Z S)MultiPeriodDiscriminatorz Wrapper class to aggregate results of multiple period discriminators. The periods are expected to be increasing prime numbers in order to maximize coverage and minimize overlap rrHrF rjperiodsc(ttfdd|D|_dS)Ncg|]}t|dqS))rr)r).0rrrr Sz5MultiPeriodDiscriminator.__init__..r.r/rQrXdiscriminators)r2rrr4rr r/P  z!MultiPeriodDiscriminator.__init__cCrwrxrrrrr rVrz$MultiPeriodDiscriminator.input_typescCrrrrrrr r]rz%MultiPeriodDiscriminator.output_typesc Cslg}g}g}g}|jD]$}||d\}} ||d\} } |||| || || q ||||fSN)r)rrq) r2r}r~rrrr discriminator score_real fmap_real score_genfmap_genrrr r9fs      z MultiPeriodDiscriminator.forward)rrj) r;r<r=rrrr/rrrrr9r>rrr4r rIs  rcsReZdZdZddedeffdd Zedd Zed d Z e d d Z Z S)DiscriminatorSTFTaA Discriminator network from EnCodec for Complex STFT input, but without dilations. Args: filters: number of filters to use in Conv2d layers lrelu_slope: Slope to use for activations. Leaky relu with slope of 0.1 or 0.2 is recommended for the stability of the feature matching loss rrjfiltersrc sxtt||_ttd|ddt||dddt||dddt||dddt||ddg|_t|ddd|_dS)Nr)rH r)rrr&)rHrHr) r.r/rQrVrOrXrrrZ)r2rrr4rr r/s     zDiscriminatorSTFT.__init__cCr)Nspec)rzrT_specrrrrrr rrzDiscriminatorSTFT.input_typescCr)Nrzrrrzrrr)scoresrtrrrrr rrzDiscriminatorSTFT.output_typescCsZg}|}|jD]}||d}||}||q|j|d}||t|d}||fS)Nrr)rrOrqrZr)r2rrtrr^rrrr r9s       zDiscriminatorSTFT.forward)rrj r;r<r=rrrr/rrrrr9r>rrr4r rws   rcsdeZdZdZdeedeeeffdd ZddZe dd Z e d d Z e d d Z ZS)MultiBandDiscriminatorSTFTa Multi-band STFT discriminator proposed in DAC (https://arxiv.org/abs/2306.06546). Computes the complex STFT for a given resolution and splits it into sub-bands, which are given to separate discriminator networks. Args: resolution: STFT resolution, provided as a tuple of 3 integers ordered (num_fft, hop_length, window_length) stft_bands: List of tuples, with each tuple having 2 float values (band_start, band_end). The floats are in the range [0, 1] representing the fraction of all stft bands. For example for n_fft=1024, the stft output has 513 dimensions. For band input [(0, 0.25), (0.25, 1.0)] it would use stft dimensions [0 through 127] and [128 through 512]. resolution stft_bandscsnt|\|_|_|_|dtj|jddt dd|D|_ |jddfdd|D|_ dS) NwindowF)periodiccSsg|]}tqSr)rrrrrr rsz7MultiBandDiscriminatorSTFT.__init__..rrcs,g|]}t|dt|dfqS)rr)rrbandn_stftrr rs,) r.r/r>rrrrP hann_windowrQrXrr)r2rrr4rr r/s z#MultiBandDiscriminatorSTFT.__init__c CsFtj||j|j|j|jdddd}t|d}tj|j|j gdd}|S)NT)r>rrr normalizedcenterreturn_complexzB fft T -> B T fftrrd) rPstftr>rrrrrlrealimag)r2rfftrrrr compute_stfts z'MultiBandDiscriminatorSTFT.compute_stftcCrrrrrrr rrz&MultiBandDiscriminatorSTFT.input_typescCs tdtgtdtggdS)Nrr) scores_list fmaps_listrrrrr rs z'MultiBandDiscriminatorSTFT.output_typesc Cs|g}g}||}t|j|jD])\}}|dddddd|d|df}||d\}} |||| q||fS)Nrr)r)rziprrrq) r2rr fmap_listrrdisc spec_bandrrtrrr r9s (  z"MultiBandDiscriminatorSTFT.forward)r;r<r=rrrrr/rrrrrr9r>rrr4r rs"   rcs`eZdZdZdeeedeeeffdd ZeddZ edd Z e d d Z Z S) MultiResolutionDiscriminatorSTFTa Multi-resolution discriminator which creates a multi-band discriminator for each input resolution. Args: resolutions: List of STFT resolutions, each resolution provided as a tuple of 3 integers ordered (num_fft, hop_length, window_length) stft_bands: List of tuples, with each tuple having 2 float values (band_start, band_end). The floats are in the range [0, 1] representing the fraction of all stft bands. For example for n_fft=1024, the stft output has 513 dimensions. For band input [(0, 0.25), (0.25, 1.0)] it would use stft dimensions [0 through 127] and [128 through 512]. resolutionsrcr)Ncr))rr)r)rrrrr rrz=MultiResolutionDiscriminatorSTFT.__init__..r)r2rrr4rr r/rz)MultiResolutionDiscriminatorSTFT.__init__cCrwrxrrrrr rrz,MultiResolutionDiscriminatorSTFT.input_typescCr)Nrrrrrrrr rrz-MultiResolutionDiscriminatorSTFT.output_typesc Csdg}g}g}g}|jD] }||d\}} ||}|| }||d\} } || }|| }q ||||fSrr) r2r}r~rrrrr score_real_i fmap_real_i score_gen_i fmap_gen_irrr r9s   z(MultiResolutionDiscriminatorSTFT.forward)r;r<r=rrrrr/rrrrr9r>rrr4r rs&   rcsPeZdZdZdeeffdd ZeddZeddZ e d d Z Z S) Discriminatorzd Wrapper class which takes a list of discriminators and aggregates the results across them. rcr,r-r)r2rr4rr r//r6zDiscriminator.__init__cCrwrxrrrrr r3rzDiscriminator.input_typescCrrrrrrr r:rzDiscriminator.output_typesc Cs\g}g}g}g}|jD]}|||d\}} } } ||7}|| 7}|| 7}|| 7}q ||||fS)Nr|r) r2r}r~rrrrrrrrrrrr r9Cs   zDiscriminator.forward) r;r<r=rrrr/rrrrr9r>rrr4r r*s  rc @seZdZeddZeddZeedej dej de ej ej ffdd Z ee d e e ed ed d e deidedej dej dej fddZee dee ed edde d e ided ej dej dej fddZdS)VectorQuantizerBasecCrNrzrrrzrrrrrrrrr rTrzVectorQuantizerBase.input_typescCstdttdtdS)Nrrrzr) dequantizedindices)rrrrrrr r[rz VectorQuantizerBase.output_typesrkrlrcCdSr-rr2rkrlrrr r9bszVectorQuantizerBase.forwardrrzrrrrrcCrr-rrrrr encodegs zVectorQuantizerBase.encoderrlrcCrr-r)r2rrlrrr decoders zVectorQuantizerBase.decodeN)r;r<r=rrrrrrPrrr9rrrrrrrrrrr rSs2  *     $rc seZdZdZd3deedeffdd ZeddZ ed d Z ed d Z ed dZ eddZ edejdejdejfddZdejdejdejfddZeedeeededdedeiddejdejdejfddZdejdejfdd Zd!ejdejfd"d#Zdejdejfd$d%Ze &d4dejdeejdeejejffd'd(Zeedeeeded)d*dd+ed,eidd4dejdeejdejfd-d.Z eed,eeeded)d*d/d0edeidd4d+ejdeejdejfd1d2Z!Z"S)5FiniteScalarQuantizeraThis quantizer is based on the Finite Scalar Quantization (FSQ) method. It quantizes each element of the input vector independently into a number of levels. Args: num_levels: number of levels for each dimension/element of the input vector eps: small regularization constant for scaling References: Mentzer et al., Finite Scalar Quantization: VQ-VAE Made Simple (https://arxiv.org/abs/2309.15505v1) MbP? num_levelsepscsttjtdg|dddtjd}t|d}|d|tj|tjd}t|d}|d|||_t d |j j t d |j t d |jt d |jt d |jdS)Nrrfr)reruz D -> 1 D 1dim_base_indexrtrzInitializing %s withz dim: %sz num_levels: %sz codebook_size: %sz eps: %s)r.r/rPcumprodrxint32rrrrdebugr5r;rer codebook_size)r2rrrr4rr r/s &    zFiniteScalarQuantizer.__init__cCs|jS)z/Returns the size of the corresponding codebook.)rproditemrrrr rsz#FiniteScalarQuantizer.codebook_sizecCs |jS)z*Returns the dimension of the input vector.)rnumelrrrr res zFiniteScalarQuantizer.dimcC|jS)z{Returns the dimension of the input vector. Keeping for compatiblitiy with the original RVQ implementation. rdrrrr codebook_dimz"FiniteScalarQuantizer.codebook_dimcCs2t|j}t|d}|j|dd}|d}|S)zReturns the codebooks entries. Note that the codebook entries are implicitly defined by the number of levels. z B -> 1 B 1Nrrf)rParangerrrr)r2rcodesrrr r s   zFiniteScalarQuantizer.codescCr)zPReturns the codebooks entries. See self.codes for more details. r rrrr codebookr zFiniteScalarQuantizer.codebookrkrlrcCst|}|||S)zqRound the input tensor to nearest integer and use a straight-through estimator for the gradient. )rProunddetach)rkrlinputs_roundedrrr rs zFiniteScalarQuantizer.roundcCsX|jdd}|d|j}t|jddkdd}||}||||}|S)z3Apply compression to the input, to limit to values.rrrg?)rrrPwheretantanh)r2rkrl output_scale output_offset input_shiftoutputrrr compresss  zFiniteScalarQuantizer.compressrrzrr rcCs2|j||d}|j||d}|jd}||}|S)Nrr)rrr)r2rkrl compressedr scalerrr inputs_to_codess   z%FiniteScalarQuantizer.inputs_to_codescCs|jd}}|||S)z;Convert values centered arouund zero to nonnegative values.rr)r2r roffsetrrr codes_to_nonnegative z*FiniteScalarQuantizer.codes_to_nonnegativecodes_nonnegativecCs|jd}}|||S)z;Convert nonnegative values to values centered arouund zero.rr)r2r rrrrr nonnegative_to_codesrz*FiniteScalarQuantizer.nonnegative_to_codescCs^|d|jkrtd|dd|jd|j||}tj||jdd}|tj S)z)Converts a code vector to a single index.rzInput code dimension % not matching the expected dimension z, input codes shape rd) rre RuntimeErrorrjrrPr6rrzr)r2r rrrr codes_to_indicess  z&FiniteScalarQuantizer.codes_to_indicesNcCs||d|jkrtd|dd|jd|j|j||d}|j|d}|dur5t||}t||}|d}||fS)NrzInput dimension r"z, inputs shape rr r)rrer#rjrr$rrr)r2rkrlrrrrr r9s    zFiniteScalarQuantizer.forwardT)optionalrrcCs|||d\}}|S)z3Convert a continuous code vector to a single index.rr)r2rkrlrrrrr rs zFiniteScalarQuantizer.encoderrcCsf|ddkrtd|dd|jdt|d}||j|j}||}|dur1t||}|S)z3Convert a single index to a continuous code vector.rrz Expected a single codebook, got z" codebooks for indices with shape .zD B T -> B D TN)rrrjrrrr!r)r2rrlr rrrr r)s    zFiniteScalarQuantizer.decode)rr-)#r;r<r=rrrrr/rrrerr r rrPrrrrrrrrrrrr!r$rrr9rrr>rrr4r rsd           $  , rcseZdZdZdedeeffdd ZeddZedd Z ed d Z e d d Z e e dee ededde deiddejdejdejfddZe e dee ededde deiddejdejdejfddZZS)GroupFiniteScalarQuantizeraSplit the input vector into groups and apply FSQ on each group separately. This class is for convenience. Since FSQ is applied on each group separately, groups can be defined arbitrarily by splitting the input vector. However, this class makes it easy to construct several groups with the same quantization num_levels. Args: num_groups: number of groups to split the input into, each group will be quantized separately using num_codebooks//num_groups codebooks codebook_dim: embedding dimension, will be split into num_groups **kwargs: parameters of FiniteScalarQuantizer References: Yang et al, HiFi-Codec: Group-residual Vector quantization for High Fidelity Audio Codec, 2023 (http://arxiv.org/abs/2305.02765). num_groupsnum_levels_per_groupc st||_t|_tjfddt|jD|_ t d|j j t d|jt d|jt dt d|jdS)Ncsg|] }tddiqS)rr)rrr8r)rr r]z7GroupFiniteScalarQuantizer.__init__..zInitialized %s withz num_groups: %dz codebook_dim: %dz num_levels_per_group: %sz codebook_dim_per_group: %d)r.r/r(rcodebook_dim_per_grouprPrQrXr+fsqsrrr5r;r)r2r(r)r8r4r*r r/Us   z#GroupFiniteScalarQuantizer.__init__cCs |j|jS)zInput vector dimension.)r,r(rrrr rf z'GroupFiniteScalarQuantizer.codebook_dimcCs |jdjS)z9Returns the size of the implicit codebook for each group.r)r-rrrrr codebook_size_per_groupkr.z2GroupFiniteScalarQuantizer.codebook_size_per_groupcCs |j|jS)z*Returns the size of the implicit codebook.)r/r(rrrr rpr.z(GroupFiniteScalarQuantizer.codebook_sizec Csx|j|jdd}gg}}t||jD]\}}|||d\}} |||| qtj|dd}tj|dd}||fS)z=Quantize each group separately, then concatenate the results.rrdrr)chunkr(rr-rqrPr8) r2rkrlinputs_groupedrrin_group fsq_groupdequantized_group indices_grouprrr r9us   z"GroupFiniteScalarQuantizer.forwardrrzrrrrrkrlrcCT|j|jdd}g}t||jD]\}}|j||d}||qtj|dd}|S)z`Input is split into groups, each group is encoded separately, then the results are concatenated.rrdrr)r0r(rr-rrqrPr8)r2rkrlr1rr2r3r5rrr rs  z!GroupFiniteScalarQuantizer.encoderrcCr6)ziInput indices are split into groups, each group is decoded separately, then the results are concatenated.rrdrr)r0r(rr-rrqrPr8)r2rrlindices_groupedrr5r3r4rrr rs  z!GroupFiniteScalarQuantizer.decode)r;r<r=rrrr/rrr/rrr9rrrrrrPrrrr>rrr4r r'Fs4       & r'cs~eZdZdZ      dded ed ed ed ed ededeffdd ZddZ e ddZ e ddZ e ddZZS) ResidualBlocka The residual block structure defined by the HiFi-GAN V1 and V2 configurations. Args: channels: Input dimension. filters: Number of channels in the residual convolutions. kernel_size: Kernel size of the residual convolutions. dilation: Dilation of the residual convolutions. dropout_rate: Dropout to apply to residuals. activation: Activation to apply in between residual convolutions. rHrrrQFrrPrrr dropout_raterO is_causalrqc stt|t||d|_t||d|_tj||_ |s3t |||||d|_ t ||||d|_ dSt |||||d|_ t ||||d|_ dS)N)rOrP)rYrZrrrqrYrZrrq)r.r8r/rMinput_activationskip_activationrPrQDropoutdropoutr input_conv skip_convrn) r2rPrrrr9rOr:rqr4rr r/s2   zResidualBlock.__init__cCs|j|jdSr-)r@rhrArrrr rhs z ResidualBlock.remove_weight_normcCrrrrrrr rszResidualBlock.input_typescCrrrrrrrr rzResidualBlock.output_typescCsF||}|j||d}||}|j||d}||}||}|SNr)r<r@r=rAr?)r2rkrl conv_input skip_inputresrrrr r9s   zResidualBlock.forward)rHrrrQFr)r;r<r=rrrrLrr/rhrrrrr9r>rrr4r r8s@ '  r8c sreZdZdZ  ddededeededed ef fd d Zd d Z e ddZ e ddZ e ddZZS)HiFiGANResBlockab Residual block wrapper for HiFi-GAN which creates a block for multiple dilations. Args: channels: Input dimension. kernel_size: Kernel size of the residual blocks. dilations: List of dilations. One residual block will be created for each dilation in the list. activation: Activation for the residual blocks. FrrPr dilationsrOr:rqcs0ttfdd|D|_dS)Nc s"g|] }t|dqS))rPrrrrOr:rqr8)rrrOrPr:rrqrr rs z,HiFiGANResBlock.__init__..r.r/rQrX res_blocks)r2rPrrIrOr:rqr4rKr r/ s   zHiFiGANResBlock.__init__cC|jD]}|qdSr-rMrhr2 res_blockrrr rh&  z"HiFiGANResBlock.remove_weight_normcCrrrrrrr r*rzHiFiGANResBlock.input_typescCrrrrrrr r1rCzHiFiGANResBlock.output_typescCs |}|jD]}|||d}q|SrD)rM)r2rkrlrrQrrr r95s zHiFiGANResBlock.forwardFrr;r<r=rrrrLrr/rhrrrrr9r>rrr4r rHs0  rHc sveZdZdZ  ddedeedeededed ef fd d Zd d Z e ddZ e ddZ e ddZZS)HiFiGANResLayeray Residual block wrapper for HiFi-GAN which creates a block for multiple kernel sizes and dilations. One residual block is created for each combination of kernel size and dilation. Args: channels: Input dimension. kernel_sizes: List of kernel sizes. dilations: List of dilations. activation: Activation for the residual layers. FrrP kernel_sizesrIrOr:rqcs0ttfdd|D|_dS)Nc s g|] }t|dqS))rPrrIrOr:rq)rH)rrrOrPrIr:rqrr rV z,HiFiGANResLayer.__init__..rL)r2rPrVrIrOr:rqr4rWr r/Js   zHiFiGANResLayer.__init__cCrNr-rOrPrrr rhcrRz"HiFiGANResLayer.remove_weight_normcCrrrrrrr rgrzHiFiGANResLayer.input_typescCr)Nrrrrrrr rnrCzHiFiGANResLayer.output_typescs*fdd|jD}t|t|}|S)Ncsg|]}|dqS)rr)rrQrlrkrr rtrz+HiFiGANResLayer.forward..)rMr6r)r2rkrl residualsrrrYr r9rszHiFiGANResLayer.forwardrSrTrrr4r rU=s0  rUceZdZdZ        dd ed eed ed ed edeedeededeffdd ZeddZ eddZ ddZ e ddZ ZS)CausalHiFiGANEncodera Causal Audio encoder created by inverting the HiFi-GAN decoder and replacing Conv1D by CausalConv1D. Args: encoded_dim: Dimension of encoder output. down_sample_rates: Rate to upsample for each decoder block. The product of the downsample rates will determine the output token rate. For example 2 * 2 * 8 * 8 = 256 samples per token. base_channels: Number of filters in the first convolution. The number of channels will be doubled after each downsample layer. in_kernel_size: Kernel size of the input convolution. out_kernel_size: Kernel size of the output convolution. resblock_kernel_sizes: List of kernel sizes to use in each residual block. resblock_dilation_sizes: List of dilations to use in each residual block. activation: Activation to use in residual and downsample layers, defaults to leaky relu. rrrrrrrHrrrrHrFrQro encoded_dimdown_sample_rates base_channelsin_kernel_sizeout_kernel_sizeresblock_kernel_sizesresblock_dilation_sizesrOrqc  s|dksJ|dks Jt||_td||| d|_|} tg|_tg|_tg|_ t |jD]9\} } t | |||d| d} |j | t || d}|j |d| }d| }t| ||| | d}|} |j |q6t || d|_t| ||| d|_dS) Nrrr;TrPrVrIrOr:rqrPr)rYrZrr%rq)r.r/rarnpre_convrQrX activationsdown_sample_conv_layers res_layers enumeraterUrqrMpost_activation post_conv)r2r`rarbrcrdrerfrOrqrYidown_sample_rate res_layeractrZrdown_sample_convr4rr r/sL         zCausalHiFiGANEncoder.__init__cCrNryrzr audio_lenrrrrrrrr rrz CausalHiFiGANEncoder.input_typescCrNrzr T_encodedrzencoded encoded_lenrrrrr rrz!CausalHiFiGANEncoder.output_typescC@|j|j|jD]}|q |jD]}|qdSr-rirhrorlrkr2rrrtrrr rh      z'CausalHiFiGANEncoder.remove_weight_normc C|}t|d}|j||d}t|j|j|j|jD]\}}}}|||d}||}||}|||d}q||}|j||d} | |fSNrr rrirrjrlrkrarnro r2rrwr~rrsrrrtrqr}rrr r9   zCausalHiFiGANEncoder.forward)r]rrrr^r_rQror;r<r=rrrrLr/rrrrhrr9r>rrr4r r\yH  ;  r\cr[)HiFiGANEncoderac Audio encoder created by inverting the HiFi-GAN decoder. Args: encoded_dim: Dimension of encoder output. down_sample_rates: Rate to upsample for each decoder block. The product of the downsample rates will determine the output token rate. For example 2 * 2 * 8 * 8 = 256 samples per token. base_channels: Number of filters in the first convolution. The number of channels will be doubled after each downsample layer. in_kernel_size: Kernel size of the input convolution. out_kernel_size: Kernel size of the output convolution. resblock_kernel_sizes: List of kernel sizes to use in each residual block. resblock_dilation_sizes: List of dilations to use in each residual block. activation: Activation to use in residual and downsample layers, defaults to leaky relu. r]rrr^r_rQrr`rarbrcrdrerfrOrqc  s|dksJ|dks Jt||_td||| d|_|} tg|_tg|_tg|_ t |jD]?\} } t | |||| d} |j | t || d}|j |d| }d| }t|| d}t| ||| || d}|} |j |q6t || d|_t| ||| d|_dS) Nrrr;rPrVrIrOrqrhrr&)rYrZrr%r)rq)r.r/rarrirQrXrjrkrlrmrUrqrMr'rnro)r2r`rarbrcrdrerfrOrqrYrprqrrrsrZrr)rtr4rr r/sN          zHiFiGANEncoder.__init__cCrrurxrrrr r>rzHiFiGANEncoder.input_typescCrryrrrrr rErzHiFiGANEncoder.output_typescCrr-rrrrr rhLrz!HiFiGANEncoder.remove_weight_normc Crrrrrrr r9TrzHiFiGANEncoder.forward)r]rrrr^r_rQrrrrr4r rrrceZdZdZ        d!d ed eededededeedeededededeffdd Ze ddZ e ddZ ddZ e dd ZZS)"CausalHiFiGANDecodera Codec decoder using the HiFi-GAN generator architecture with Causal Convolutions. Args: input_dim: Input dimension. up_sample_rates: Rate to upsample for each decoder block. The product of the upsample rates should be the same as the overall downsample rate for your encoder. For example, a symmetric encoder/decoder can be created with encoder downsample rates [2, 2, 8, 8] and decoder upsample rates [8, 8, 2, 2]. base_channels: Number of filters in the first convolution. The number of channels will be cut in half after each upsample layer. in_kernel_size: Kernel size of the input convolution. out_kernel_size: Kernel size of the output convolution. resblock_kernel_sizes: List of kernel sizes to use in each residual block. resblock_dilation_sizes: List of dilations to use in each residual block. activation: Activation to use in residual and upsample layers, defaults to leaky relu. output_activation: Activation to apply to output. To produce a valid audio signal, it should output values in the range [-1.0, 1.0]. Supports "tanh" and "clamp". rrrrrrrHr^r_rQrroTr up_sample_ratesrbrcrdrerfrOoutput_activationrqn_groups_equal_to_out_channelsc  s@|dksJ|dks Jt||_t|||| d|_|} tg|_tg|_tg|_ t |jD]=\} }| d}d|}t || d}|j |t | |||| rW|ndd}|} |j |t| |||d| d}|j |q6t || d|_t| d|| d|_| d krt|_dS| d krt|_dStd | ) Nrr;rrhrrYrZrr%r[Trgrr7 Invalid audio output activation )r.r/rrnrirQrXrjup_sample_conv_layersrlrmrMrqrXrUrnroTanhout_activationr rr2r rrbrcrdrerfrOrrqrrYrpup_sample_raterZrrsup_sample_convrrr4rr r/sV           zCausalHiFiGANDecoder.__init__cCrNrzrzrrrrrr rrz CausalHiFiGANDecoder.input_typescCrrurxrrrr rrz!CausalHiFiGANDecoder.output_typescC6|j|jD]}|q|jD]}|qdSr-rirhrrlr2rrrrrr rh     z'CausalHiFiGANDecoder.remove_weight_normc C|}|j||d}t|j|j|j|jD]\}}}}||}||}|||d}|||d}q||}|j||d}||} t | d} | |fSNrz B 1 T -> B T rirrjrlrrrnrorr r2rkrlrwrrsrrrrrrrr r9    zCausalHiFiGANDecoder.forward) rrrrHr^r_rQrroTr;r<r=rrrrLrr/rrrrhrr9r>rrr4r rksT    C  rcr)"HiFiGANDecodera Codec decoder using the HiFi-GAN generator architecture. Default parameters match the HiFi-GAN V1 configuration for 22.05khz. Args: input_dim: Input dimension. up_sample_rates: Rate to upsample for each decoder block. The product of the upsample rates should be the same as the overall downsample rate for your encoder. For example, a symmetric encoder/decoder can be created with encoder downsample rates [2, 2, 8, 8] and decoder upsample rates [8, 8, 2, 2]. base_channels: Number of filters in the first convolution. The number of channels will be cut in half after each upsample layer. in_kernel_size: Kernel size of the input convolution. out_kernel_size: Kernel size of the output convolution. resblock_kernel_sizes: List of kernel sizes to use in each residual block. resblock_dilation_sizes: List of dilations to use in each residual block. activation: Activation to use in residual and upsample layers, defaults to leaky relu. output_activation: Activation to apply to output. To produce a valid audio signal, it should output values in the range [-1.0, 1.0]. Supports "tanh" and "clamp". rrrrHr^r_rQrrFr rrbrcrdrerfrOrrqrc s>|dksJ|dks Jt||_t|||| d|_|} tg|_tg|_tg|_ t |jD]<\} }| d}d|}t || d}|j |t | |||| rW|ndd}|} |j |t| |||| d}|j |q6t || d|_t| d|| d|_| dkrt|_dS| d krt|_dStd | ) Nrr;rrhrrrrr7r)r.r/rrrirQrXrjrrlrmrMrqrrUrnrorrr rrr4rr r/sT           zHiFiGANDecoder.__init__cCrrrrrrr rErzHiFiGANDecoder.input_typescCrrurxrrrr rLrzHiFiGANDecoder.output_typescCrr-rrrrr rhSrz!HiFiGANDecoder.remove_weight_normc Crrrrrrr r9ZrzHiFiGANDecoder.forward) rrrrHr^r_rQrrFrrrr4r rsT    B  rc s^eZdZdZddededededef fd d Zed d Zed dZ e ddZ Z S)MelSpectrogramProcessorzM Wrapper interface for computing mel spectrogram for codec training. Prrrrmel_dim log_guardcstt|||_||_tdid|ddd|ddddd|d |d d d d d |ddddddd|dddddddd|_dS)Nrhighfreqfeaturespad_tor exact_padT n_window_sizen_window_stride window_sizeF window_strider> mag_powerrr4log_zero_guard_typeaddlog_zero_guard_valuemel_normr9preemphditherrr)r.rr/rrr preprocessor)r2rrrrrr4rr r/usP      z MelSpectrogramProcessor.__init__cCrrurxrrrr rrz#MelSpectrogramProcessor.input_typescCr)N)rzrrrz)rspec_len)rrrrrrrr rrz$MelSpectrogramProcessor.output_typescCs|j||d\}}||fS)N) input_signalr{)r)r2rrwrrrrr r9szMelSpectrogramProcessor.forward)rrrrrr4r rps$  rcseZdZdZ       dd ed ed ed ed ededededeffdd ZddZe ddZ e ddZ e ddZ ZS) ResNetEncoderaB Residual network which uses HiFi-GAN residual blocks to encode spectrogram features without changing the time dimension. Args: in_channels: input dimension out_channels: output dimension num_layers: number of residual blocks to use hidden_channels: encoder hidden dimension filters: number of filters in residual block layers kernel_size: kernel size in residual block convolutions dropout_rate: Optional dropout rate to apply to residuals. activation: Activation to use, defaults to leaky relu. rrrCrHrjrQrrYrZ num_layershidden_channelsrrr9rOrqc sltt|t|d|_tfddt|D|_t d|_ t|d|_ dS)Nr;c s g|] }tdqS))rPrrr9rOrqrJrrOr9rrrrqrr rrXz*ResNetEncoder.__init__..rh) r.rr/rrirQrXr+rlrMrnro) r2rYrZrrrrr9rOrqr4rr r/s    zResNetEncoder.__init__cCs,|j|j|jD]}|q dSr-)rirhrorl)r2rrrrr rhs    z ResNetEncoder.remove_weight_normcCrrrrrrr rrzResNetEncoder.input_typescCr)Nr}rrBrrrr rrCzResNetEncoder.output_typescCsB|j||d}|jD]}|||d}q ||}|j||d}|SrD)rirlrnro)r2rkrlr}rrrrr r9s   zResNetEncoder.forward)rrrCrHrjrQr)r;r<r=rrrrLr/rhrrrrr9r>rrr4r rsF  #  rcsXeZdZdZdedeffdd ZddZedd Zed d Z e d d Z Z S)FullBandMelEncoderaE Encoder which encodes the entire mel spectrogram with a single encoder network. Args: mel_processor: MelSpectrogramProcessor or equivalent class instance for computing the mel spectrogram from input audio. encoder: ResNetEncoder or equivalent class for encoding the mel spectrogram. mel_processorencodercstt|||_||_dSr-)r.rr/rr)r2rrr4rr r/s zFullBandMelEncoder.__init__cCs|jdSr-)rrhrrrr rhsz%FullBandMelEncoder.remove_weight_normcCrrurxrrrr r rzFullBandMelEncoder.input_typescCrN)rzrr{rzr|rrrrr r rzFullBandMelEncoder.output_typescCs(|j||d\}}|j||d}||fS)Nrvr)rr)r2rrwrrr}rrr r9 szFullBandMelEncoder.forward) r;r<r=rrr/rhrrrrr9r>rrr4r rs   rcseZdZdZdeeeefdeffdd Ze dedeeeeffddZ d d Z e d d Z e d dZeddZZS)MultiBandMelEncodera Encoder which splits mel spectrogram into bands and encodes each using separate residual networks. Args: mel_bands: List of mel spectrogram bands to encode. Each list element is tuple of 2 elements with the start and end index of the mel features to use. mel_processor: MelSpectrogramProcessor or equivalent class instance for computing the mel spectrogram from input audio. encoder_kwargs: Arguments for constructing encoder for each mel band. mel_bandsrc sXtt||j|j|d||_||_dd|jD}tfdd|D|_ dS)N)rrcSsg|] }|d|dqSrrrrrr r( sz0MultiBandMelEncoder.__init__..csg|] }tdd|iqS)rYr)r)rband_dimencoder_kwargsrr r* r+) r.rr/validate_mel_bandsrrrrQrXencoders)r2rrr band_dimsr4rr r/# s zMultiBandMelEncoder.__init__rcCs\tj|gtd}|D] }d||d|d<q t|s,t|}td|d|ddS)NrtTrrzMel bands must cover all z dimensions. Missing r&)nprorallrr)rr mel_dims_usedr missing_dimsrrr r- s z&MultiBandMelEncoder.validate_mel_bandscCrNr-)rrh)r2rrrr rh9 rRz&MultiBandMelEncoder.remove_weight_normcCrrurxrrrr r= rzMultiBandMelEncoder.input_typescCrrrrrrr rD rz MultiBandMelEncoder.output_typesc Csx|j||d\}}g}t|j|jD]\\}}}|dd||ddf} || |d} || qtj|dd} | |fS)Nrvrrrd)rrrrrqrPr8) r2rrwrroutputs band_startband_endrrband_outr}rrr r9K s  zMultiBandMelEncoder.forward)r;r<r=rrrrrr/rrrhrrrrr9r>rrr4r r s"     rrKr-)Zr_rabcrrtypingrrrrnumpyrrPtorch.nnrQtorch.nn.functionalrroeinopsr transformersr nemo.collections.asr.modulesr #nemo.collections.common.parts.utilsr r r rnemo.core.classes.commonrnemo.core.classes.modulernemo.core.neural_types.elementsrrrrrr"nemo.core.neural_types.neural_typer nemo.utilsrrJrIrrrrr!r$r'r*r+r?rrLrrrrrModulerMrXrnrrrrrrrrrrrr'r8rHrUr\rrrrrrrrrrr s           . ]  !:7c0'+B.9D6)-GpK<<yy2N'