o }oip @s&ddlmZmZmZddlZddlmZddlmmZ ddl m Z m Z ddlm Z ddlmZddlmZddlmZmZmZmZmZmZddlmZdd lmZdd lmZdd lm Z m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'dd l(m)Z)ddl*m+Z+GdddeZ,GdddeZ-GdddeZ.GdddeZ/GdddeZ0GdddeZ1de de de2ddfdd Z3d>d"e d#e4d$e2de fd%d&Z5d'e d(e de fd)d*Z6d+e d,e4de fd-d.Z7d?d+e d0e4d1e4dee e ffd2d3Z8d4e d5e fd6d7Z9e+Gd8d9d9eZ:Gd:d;d;eZ;Gdr3r3r4 output_typesR zSEANetResnetBlock.output_typescCs"|j|j|jdSN)r+remove_weight_normr,r.r>r3r3r4rHXs  z$SEANetResnetBlock.remove_weight_normcCsP||}|j||d}||}|j||d}|j||d|}t||}|S)Nr9)r*r,r-r.r+r)r/r:r;resrBr3r3r4forward]s   zSEANetResnetBlock.forward)r) __name__ __module__ __qualname__intstrr)propertyr?rErHrrJ __classcell__r3r3r1r4rAs  rc sVeZdZddedededeffdd Zed d Zed d Z e d dZ Z S) SEANetRNNlstmFdim num_layersrnn_typeuse_skipcs^t||_|dkrtjj|||d|_dS|dkr(tjj|||d|_dStd|)NrS) input_size hidden_sizerUgruzUnknown RNN type ) r(r)rWtorchnnLSTMrnnGRU ValueError)r/rTrUrVrWr1r3r4r)js zSEANetRNN.__init__cCr5)Nr6r7Tr6r9r<r>r3r3r4r?tr@zSEANetRNN.input_typescCrA)NrBrarDr>r3r3r4rE{rFzSEANetRNN.output_typescCsdt|d}tjjj||ddd}||\}}tjjj|dd\}}|jr+||}t|d}|S)NzB C T -> B T CTF)lengths batch_firstenforce_sorted)rdzB T C -> B C T)rr\utilsr^pack_padded_sequencecpupad_packed_sequencerW)r/r:r; packed_inputs packed_out_rBr3r3r4rJs   zSEANetRNN.forward)rSF) rKrLrMrNrOboolr)rPr?rErrJrQr3r3r1r4rRis   rRcseZdZ         dd eed ed ed ed ededededeffdd ZeddZ eddZ ddZ e ddZ ZS) SEANetEncoderr! rr!rSTdown_sample_rates base_channelsin_kernel_sizeout_kernel_size encoded_dimr rnn_layersrVrnn_skipc  s |dksJ|dks Jt||_td||d|_|} tg|_tg|_tg|_ t |jD]5\} } t | d} |j | |j t || dd| }d| }t| ||| t|| d}|} |j |q5t || d|_t| ||| d|_t| ||d|_dS) Nrr"r#rr r!)r$r%r&stridepaddingrTrUrVrW)r(r)rvr r+r\ ModuleList res_blocksdown_sample_conv_layers activations enumeraterappendr rr-rRr^ post_conv)r/rvrwrxryrzrr{rVr|r$idown_sample_rate res_blockr%r&down_sample_convr1r3r4r)s6       zSEANetEncoder.__init__cCr5Nr6T_audior6)audio audio_lenrrr=rr>r3r3r4r?r@zSEANetEncoder.input_typescCr5)Nr6D T_encodedr6)encoded encoded_lenrrr=rr>r3r3r4rEr@zSEANetEncoder.output_typescCs@|j|j|jD]}|q |jD]}|qdSrG)r+rHrrr)r/rrr3r3r4rHs      z SEANetEncoder.remove_weight_normc Cs|}t|d}|j||d}t|j|j|j|jD]\}}}}|||d}||}||}|||d}q|j||d}||}|j ||d} | |fS)Nz B T -> B 1 Tr9) rr+ziprrrvrr^r-r) r/rrrrBrrrrrr3r3r4rJs   zSEANetEncoder.forward) rorsrtrtrurr!rSTrKrLrMrrNrOrmr)rPr?rErHrrJrQr3r3r1r4rnsH  -  rncseZdZ         dd eed ed ed edededededeffdd ZeddZ eddZ ddZ e ddZ ZS) SEANetDecoderrrrqrpr!rtr'rurr!rSTup_sample_ratesrwrxryrzrr{rVr|c s |dksJ|dks Jt||_t|||d|_t|||| d|_|} tg|_ tg|_ tg|_ t |jD]1\} } |j t|| d| d} d| }t| | || d}| } |j |t| d}|j |q>t|| d|_t| d|d|_t|_dS) Nrr#rr r!)r$r%r&r~r}r")r(r)rr r+rRr^r\rrup_sample_conv_layersrrrr r rr-rTanhout_activation)r/rrwrxryrzrr{rVr|r$rup_sample_rater%r&up_sample_convrr1r3r4r)s6       zSEANetDecoder.__init__cCs"tdtgttdtgdS)Nrr6r9rr>r3r3r4r?s zSEANetDecoder.input_typescCr5rrr>r3r3r4rE#r@zSEANetDecoder.output_typescCs6|j|jD]}|q|jD]}|qdSrG)r+rHrr)r/rrr3r3r4rH*s     z SEANetDecoder.remove_weight_normc Cs|}|j||d}|j||d}t|j|j|j|jD]\}}}}||}||}|||d}|||d}q||}|j||d}| |} t | d} | |fS)Nr9z B 1 T -> B T) r+r^rrrrrr-rrr) r/r:r;rrBrrrrrr3r3r4rJ1s    zSEANetDecoder.forward) rrrtr'rurr!rSTrr3r3r1r4rsH  -  rcsLeZdZd fdd ZddZeddZedd Zed d Z Z S) DiscriminatorSTFT皙?c st|\|_|_|_|dtj|jddt ||_ t t ddddt ddddd d t dddd d d t dddd d d t ddd dg|_ t ddd d|_dS)NwindowF)periodicr!rs)r' )r&)r"r")r"r!)r&dilationr~)r!r")rpr")r'r'r")r(r)n_fft hop_length win_lengthregister_bufferr[ hann_windowr\ LeakyReLUrrr conv_layers conv_post)r/ resolution lrelu_sloper1r3r4r)Is     zDiscriminatorSTFT.__init__c CsFtj||j|j|j|jdddd}t|d}tj|j|j gdd}|S)NT)rrrr normalizedcenterreturn_complexzB fft T -> B 1 T fftr"rT) r[stftrrrrrcatrealimag)r/rrBr3r3r4r[s zDiscriminatorSTFT.stftcCrA)Nrrrrr>r3r3r4r?lrFzDiscriminatorSTFT.input_typescCstdttdtgdS)Nr6r7T_specr6rrr7)scoresfmaprDr>r3r3r4rErs  zDiscriminatorSTFT.output_typescCs`g}||}|jD]}||d}||}||q |j|d}||t|d}||fS)Nr:zB 1 T C -> B C T)rrrrrr)r/rrrBconvrr3r3r4rJys        zDiscriminatorSTFT.forward)r) rKrLrMr)rrPr?rErrJrQr3r3r1r4rHs  rcsBeZdZfddZeddZeddZeddZZ S) MultiResolutionDiscriminatorSTFTcs$ttdd|D|_dS)NcSsg|]}t|qSr3)r).0rIr3r3r4 sz=MultiResolutionDiscriminatorSTFT.__init__..)r(r)r\rdiscriminators)r/ resolutionsr1r3r4r)s z)MultiResolutionDiscriminatorSTFT.__init__cCstdttdtdS)Nr) audio_real audio_genrr>r3r3r4r?  z,MultiResolutionDiscriminatorSTFT.input_typescCs:tdtgtdtgtdtggtdtggdS)Nrr) scores_real scores_gen fmaps_real fmaps_genrDr>r3r3r4rEs  z-MultiResolutionDiscriminatorSTFT.output_typesc Cslg}g}g}g}|jD]$}||d\}} |||| ||d\} } || || q ||||fS)N)r)rr) r/rrrrrrdisc score_real fmap_real score_genfmap_genr3r3r4rJs      z(MultiResolutionDiscriminatorSTFT.forward) rKrLrMr)rPr?rErrJrQr3r3r1r4rs   r moving_avgnewdecayreturncCs|j|j|d|ddS)Nr")alpha)datamul_add_rrrr3r3r4 _ema_inplacesrh㈵>r: n_categoriesepsiloncCs$|}|||||}||SrG)sum)r:rr input_sumsmoothedr3r3r4_laplace_smoothingsrinput1input2cCsBt|d}|djdddd|||djddd}|S)z Compute pairwise L2 distance between two input tensors Args: input1: [B, D] first tensor. input2: [N, D] second tensor. Returns: [(B, D)] tensor of distances. z N D -> D Nr!r"T)keepdimr)rpowr)rr distancesr3r3r4_compute_distancess 4rsamples num_samplecCsR|j}|jd}||krtj||dd|}||Stjd||f|d}||S)a Randomly sample from the input batch. Args: samples: [B, D] tensor with features to sample. num_sample: Number of samples to draw. If the value is less than or equal to B, then the samples will be unique. If the value is greater than B, then samples will be drawn with replacement. Returns: Tensor with num_sample values randomly sampled from the input batch. rdeviceN)lowhighsizer)rshaper[randpermrandint)rrr total_samplesindicesr3r3r4_sample_vectorss r num_clusters num_itersc Cs|dksJ|jd}t||d}t|D]D}t||}|jddj}t|d|d}tj||d} t | d} |j |||j d } | j d||d | tj | dd } t| dk|| }q|| fS) aE K-means clustering algorithm. Args: samples: [B, D] tensor with features to cluster num_clusters: K, the number of clusters. num_iters: Number of iterations of K-means to run. Returns: [K, D] cluster means and [K] bins counting how many input samples belong to each cluster rr"rrrzB -> B D)r) minlength K -> K ())dtype)rTindexsrc)min)rrrangerrrrr[bincountr new_zerosr scatter_add_clampwhere) rrr input_dimmeansrldistsbucketsbuckets_repeated bin_countsbin_counts_expanded new_meansr3r3r4_k_meanss     rtensorrccCs>|j\}}}t||djdd|}|t|dk}||S)z Mask 3d tensor with time on 1st axis. Args: tensor: tensor of shape (B, T, D) lengths: LongTensor of shape (B,) Returns: Masked Tensor (B, T, D) r"rz b -> b 1 1)rr[onescumsumtype_asr)rrc batch_size max_lengthsrlmaskr3r3r4_mask_3d s rc s<eZdZdZ   d+dedededeed eef fd d Zej j d d Z de ddfddZ de de ddfddZde de fddZde de fddZeddZeddZeddZeed eeed!ed"ded#eid$d%d&Zeed#eeed!ed'd(ed eid$d)d*ZZS),EuclideanCodebooka Codebook with Euclidean distance. Args: codebook_size: Number of codes to use. codebook_dim: Dimension of each code. decay: Decay for exponential moving average over the codebooks. threshold_ema_dead_code: Threshold for dead code expiration. During every iteration, replace codes with exponential moving average cluster size less than threshold with randomly selected values from the current batch. kmeans_iters: Optional int, if provided codes will be initialized from the centroids learned from kmeans_iters iterations of k-means clustering on the first training batch. Gz?@2 codebook_size codebook_dimrthreshold_ema_dead_code kmeans_iterscst||_|rtjt||}nt||}||_ ||_ ||_ | dt | g| dt|| d|| d|dS)N initialized cluster_sizecodes codes_avg)r(r)rr\initkaiming_uniform_r[emptyzerosrr rrrclone)r/rrrrr r#r1r3r4r),s   zEuclideanCodebook.__init__cCsr|jrdSt||j|jd\}}|jj||jj||j j||jjt dgt | dS)N)rrrT) r!rrr r#rcopy_r$r)r"rrbuffers)r/rr#r"r3r3r4 _init_codesFszEuclideanCodebook._init_codesr:rNcCsj|jsdS|j|jk}t|sdSt||jd}t|d}t|||j}|jj |t | dS)Nrr) rr"r[anyrrrrr#rr*rr+)r/r: expired_codesrmodified_codesr3r3r4 _expire_codesRs   zEuclideanCodebook._expire_codesrcCst||j|j}t|d}|d}t|j||j d||}t|j ||j dt |j|jd}t|d}|j |}|j j |dS)Nz B N -> N Br"r)rz N -> N ())Fone_hotrtyperrrrr"rr$rr#rr*)r/r:r code_onehot code_countscode_sumcluster_size_smoothedcodes_normalizedr3r3r4 _update_codesas    zEuclideanCodebook._update_codescCst||j}|jddj}|S)Nr"r)rr#rr)r/r:distrr3r3r4 _quantizeps zEuclideanCodebook._quantizecCst||j}|SrG)r1 embeddingr#)r/r dequantizedr3r3r4 _dequantizewszEuclideanCodebook._dequantizecCr5)Nr6rbrr6r9rr>r3r3r4r?|r@zEuclideanCodebook.input_typescCstdttdtdS)Nr?r6rb)r=r)rrrr>r3r3r4rErzEuclideanCodebook.output_typescCs|t|d}|||j|d}|j|jdd}|j|d}|jr0|j|d|j||dt ||}t ||}||fS)NB T D -> (B T) Drr)r:r) rr,r;viewrr>trainingr0r9rr)r/r:r; input_flat indices_flatrr=r3r3r4rJs       zEuclideanCodebook.forwardr?r6r9r@r?rEcCs8t|d}|j|d}|j|jdd}t||}|S)NrArrB)rr;rDrr)r/r:r;rFrGrr3r3r4encodes   zEuclideanCodebook.encoderr;r=cCs|j|d}t||}|S)NrC)r>r)r/rr;r=r3r3r4decodes zEuclideanCodebook.decode)rrr)rKrLrM__doc__rNfloatrr)r[jitignorer,rr0r9r;r>rPr?rErrJrrr=rrrIrKrQr3r3r1r4rsT        rcseZdZdZ     d"deded ed ed eed eef fd d ZeddZ e de de de e e effddZ e edeeededdedeidde de de fddZe edeeededdedeidde de de fd d!ZZS)#ResidualVectorQuantizera Residual vector quantization (RVQ) algorithm as described in https://arxiv.org/pdf/2107.03312.pdf. Args: num_codebooks: Number of codebooks to use. codebook_size: Number of codes to use for each codebook. codebook_dim: Dimension of each code. decay: Decay for exponential moving average over the codebooks. threshold_ema_dead_code: Threshold for dead code expiration. During every iteration, replace codes with exponential moving average cluster size less than threshold with randomly selected values from the current batch. kmeans_iters: Optional int, if provided codes will be initialized from the centroids learned from kmeans_iters iterations of k-means clustering on the first training batch. rurrr num_codebooksrrrrr csBt|_t|_tfddt|D|_dS)Nc sg|] }tdqS))rrrrr )rrrlrrrr rr3r4rsz4ResidualVectorQuantizer.__init__..) r(r)rr commit_loss_fnr\rr codebooks)r/rRrrrrr r1rTr4r)s  z ResidualVectorQuantizer.__init__cC$tdttdttdtdSNr6rrbrr6rbr3)r=r commit_lossrrrrr>r3r3r4rE   z$ResidualVectorQuantizer.output_typesr:r;rc Csd}t|d}g}t|}|jD]=}|||d\}} |jrA|} |jt|dt| d|d} || }|| }|||}n||}||}|| qt|} t|d}|| |fS)NgB D T -> B T Dr9B T D -> B D T) predictedtarget target_len) rr[ zeros_likerVrEdetachrUrstack) r/r:r;r[residual index_listr=codebook dequantized_i indices_idequantized_i_const commit_loss_irr3r3r4rJs,       zResidualVectorQuantizer.forwardrYr6r9rrZrHc CsVt|d}g}|jD]}|j||d}|j||d}||}||q t|}|S)Nr^r9rJ)rrVrIrKrr[re) r/r:r;rfrgrhrjrirr3r3r4rIs    zResidualVectorQuantizer.encoderJr=cCs^tj|jd|jd|jg|jd}t||jD]\}}|j||d}||}qt|d}|S)Nr"r!rrJr_) r[r(rrrrrVrKr)r/rr;r=codebook_indicesrhrir3r3r4rK%s $   zResidualVectorQuantizer.decode)rQrurrr)rKrLrMrLrNrMrr)rPrErrrrJrrr=rrrIrKrQr3r3r1r4rPsN "    rPcseZdZdZdededeffdd ZeddZed d Zed d Z e d dZ e e de e ededde deiddededefddZe e dee ededde de iddededefddZZS)GroupResidualVectorQuantizera7Split the input vector into groups and apply RVQ on each group separately. Args: num_codebooks: total number of codebooks num_groups: number of groups to split the input into, each group will be quantized separately using num_codebooks//num_groups codebooks codebook_dim: embedding dimension, will be split into num_groups **kwargs: parameters of ResidualVectorQuantizer References: Yang et al, HiFi-Codec: Group-residual Vector quantization for High Fidelity Audio Codec, 2023 (http://arxiv.org/abs/2305.02765). rR num_groupsrc st|_|_|_tjfddtjD_ t dj j t djt djt djt djt djdS) Ncs$g|]}tdjjdqS))rRrr3)rPnum_codebooks_per_groupcodebook_dim_per_grouprSkwargsr/r3r4rNsz9GroupResidualVectorQuantizer.__init__..zInitialized %s withz num_codebooks: %dz num_groups: %dz codebook_dim: %dz num_codebooks_per_group: %dz codebook_dim_per_group: %d)r(r)rRrorr[r\rrrvqsrdebugr2rKrprq)r/rRrorrsr1rrr4r)Es   z%GroupResidualVectorQuantizer.__init__cC6|j|jdkrtd|jd|jd|j|jS)z#Number of codebooks for each group.rznum_codebooks (#) must be divisible by num_groups ())rRror`r>r3r3r4rp]s  z4GroupResidualVectorQuantizer.num_codebooks_per_groupcCrv)z&Input vector dimension for each group.rzcodebook_dim (rwrx)rror`r>r3r3r4rqgs z3GroupResidualVectorQuantizer.codebook_dim_per_groupcCrWrXr\r>r3r3r4rEpr]z)GroupResidualVectorQuantizer.output_typesc Cs|j|jdd}gg}}d}t||jD]\}}|||d\} } } || || || 7}qtj|dd}tj|dd}|||fS)z=Quantize each group separately, then concatenate the results.r"rrr9)chunkrorrtrr[r) r/r:r;inputs_groupedr=rr[in_group rvq_groupdequantized_group indices_groupcommit_loss_groupr3r3r4rJxs     z$GroupResidualVectorQuantizer.forwardrYr6r9rrZrHr:r;rcCT|j|jdd}g}t||jD]\}}|j||d}||qtj|dd}|S)z`Input is split into groups, each group is encoded separately, then the results are concatenated.r"rr9r)ryrorrtrIrr[r)r/r:r;rzrr{r|r~r3r3r4rIs  z#GroupResidualVectorQuantizer.encoderJr=cCr)ziInput indices are split into groups, each group is decoded separately, then the results are concatenated.rrrJr")ryrorrtrKrr[r)r/rr;indices_groupedr=r~r|r}r3r3r4rKs  z#GroupResidualVectorQuantizer.decode)rKrLrMrLrNr)rPrprqrErrJrrr=rrrrIrKrQr3r3r1r4rn8s4         rn)r)r)=typingrrrr[torch.nnr\torch.nn.functional functionalr1einopsrrr#nemo.collections.common.parts.utilsr,nemo.collections.tts.losses.audio_codec_lossr 0nemo.collections.tts.modules.audio_codec_modulesr r r r rr,nemo.collections.tts.parts.utils.distributedrnemo.core.classes.commonrnemo.core.classes.modulernemo.core.neural_types.elementsrrrrrr"nemo.core.neural_types.neural_typer nemo.utilsrnemo.utils.decoratorsrrrRrnrrrrMrrNrrrrrrrPrnr3r3r3r4sB%            ()\ZD($& |