o æS™iŸ3ã@södZddlZddlmZmZddlZddlmZddlmm Z ddl m Z dej dej d ej fd d „Zd efd d„Zd%dedefdd„Zdefdd„Zdefdd„Zd&dedefdd„ZGdd „d ejƒZGd!d"„d"ejƒZGd#d$„d$ejƒZdS)'z(Core vector quantization implementation.éN)Ú rearrangeÚrepeat)Únné)ÚdistribÚvalÚdÚreturncCs|dur|S|S©N©)rrr r úP/home/ubuntu/.local/lib/python3.10/site-packages/encodec/quantization/core_vq.pyÚdefault,sr ÚdecaycCs|j |¡j|d|ddS)Né)Úalpha)ÚdataÚmul_Úadd_)Ú moving_avgÚnewrr r r Ú ema_inplace0srçñh㈵øä>Ú n_categoriesÚepsiloncCs||| ¡||Sr )Úsum)Úxrrr r r Úlaplace_smoothing4srÚshapecGst |¡}tj |¡|Sr )ÚtorchÚemptyrÚinitÚkaiming_uniform_)rÚtr r r Ú uniform_init8s  r#ÚnumcCsT|jd|j}}||krtj||dd|…}||Stjd||f|d}||S)Nr©Údevice)rr&rÚrandpermÚrandint)Úsamplesr$Ú num_samplesr&Úindicesr r r Úsample_vectors>s þr,é Ú num_clustersÚ num_itersc CsÎ|jd|j}}t||ƒ}t|ƒD]P}t|dƒt|dƒ}|djdd }|jddj} tj | |d} | dk} |   | d¡} | j |||d } |   dt | d |d |¡| | d } t | d || ¡}q|| fS) Néÿÿÿÿz n d -> n () dz c d -> () c dr©Údim)Ú minlengthrr)Údtypezn -> n d)r©.N)rr4r,ÚrangerrÚmaxr+rÚbincountÚ masked_fillÚ new_zerosÚ scatter_add_rÚwhere)r)r.r/r2r4ÚmeansÚ_ÚdiffsÚdistsÚbucketsÚbinsÚ zero_maskÚbins_min_clampedÚ new_meansr r r ÚkmeansIs    ÿ  rFcs¢eZdZdZ     d$deded ed ed ed ed ef‡fdd„ Zejj dd„ƒZ dd„Z dd„Z dd„Z dd„Zdd„Zdd„Zdd„Zd d!„Zd"d#„Z‡ZS)%ÚEuclideanCodebookaJCodebook with Euclidean distance. Args: dim (int): Dimension. codebook_size (int): Codebook size. kmeans_init (bool): Whether to use k-means to initialize the codebooks. If set to true, run the k-means algorithm on the first training batch and use the learned centroids as initialization. kmeans_iters (int): Number of iterations used for k-means algorithm at initialization. decay (float): Decay for exponential moving average over the codebooks. epsilon (float): Epsilon value for numerical stability. threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes that have an exponential moving average cluster size less than the specified threshold with randomly selected vector from the current batch. Fr-ç®Gáz®ï?rrr2Ú codebook_sizeÚ kmeans_initÚ kmeans_itersrrÚthreshold_ema_dead_codec sˆtƒ ¡||_|s tntj}|||ƒ} ||_||_||_||_ |  dt  | g¡¡|  dt |¡¡|  d| ¡|  d|   ¡¡dS)NÚinitedÚ cluster_sizeÚembedÚ embed_avg) ÚsuperÚ__init__rr#rÚzerosrIrKrrLÚregister_bufferÚTensorÚclone) Úselfr2rIrJrKrrrLÚinit_fnrO©Ú __class__r r rRqs   zEuclideanCodebook.__init__cCst|jrdSt||j|jƒ\}}|jj |¡|jj | ¡¡|j j |¡|jj t   dg¡¡t   | ¡¡dS)NT)rMrFrIrKrOrÚcopy_rPrVrNrrUrÚbroadcast_tensorsÚbuffers)rWrrOrNr r r Ú init_embed_‹szEuclideanCodebook.init_embed_cCs.t |dt||jƒ|j¡}|jj |¡dS)Nr5)rr<r,rIrOrr[)rWr)ÚmaskÚmodified_codebookr r r Úreplace_˜sÿzEuclideanCodebook.replace_cCsR|jdkrdS|j|jk}t |¡sdSt|dƒ}|j||dt | ¡¡dS)Nrú... d -> (...) d)r_) rLrNrÚanyrrarr\r])rWÚ batch_samplesÚ expired_codesr r r Ú expire_codes_žs    zEuclideanCodebook.expire_codes_cCst|dƒ}|S)Nrb)r)rWrr r r Ú preprocessªó zEuclideanCodebook.preprocesscCsR|j ¡}| d¡jdddd||| d¡jddd }|jddj}|S)NrrT)Úkeepdimrr0r1)rOr"Úpowrr7r+)rWrrOÚdistÚ embed_indr r r Úquantize®s  ÿþÿzEuclideanCodebook.quantizecCs|j|dd…ŽS)Nr0)Úview)rWrlrr r r Úpostprocess_emb¸sz!EuclideanCodebook.postprocess_embcCst ||j¡}|Sr )ÚFÚ embeddingrO©rWrlrmr r r Ú dequantize»szEuclideanCodebook.dequantizecCs*|j}| |¡}| |¡}| ||¡}|Sr )rrgrmro)rWrrrlr r r Úencode¿s    zEuclideanCodebook.encodecCs| |¡}|Sr )rsrrr r r ÚdecodeÉrhzEuclideanCodebook.decodec CsÞ|j|j}}| |¡}| |¡| |¡}t ||j¡ |¡}|  ||¡}|  |¡}|j rk|  |¡t |j| d¡|jƒ| ¡|}t |j| ¡|jƒt|j|j|jƒ|j ¡}|j| d¡} |jj | ¡||fS)Nrr)rr4rgr^rmrpÚone_hotrIÚtyperorsÚtrainingrfrrNrrr"rPrrÚ unsqueezerOrr[) rWrrr4rlÚ embed_onehotrmÚ embed_sumrNÚembed_normalizedr r r ÚforwardÍs&       ÿÿzEuclideanCodebook.forward)Fr-rHrr)Ú__name__Ú __module__Ú __qualname__Ú__doc__ÚintÚfloatrRrÚjitÚignorer^rarfrgrmrorsrtrur}Ú __classcell__r r rYr rGbsBøþýüûúùø     rGcs€eZdZdZ       dd ed ed ejed ed ededededef‡fdd„ Z e dd„ƒZ dd„Z dd„Z dd„Z‡ZS)ÚVectorQuantizationa€Vector quantization implementation. Currently supports only euclidean distance. Args: dim (int): Dimension codebook_size (int): Codebook size codebook_dim (int): Codebook dimension. If not defined, uses the specified dimension in dim. decay (float): Decay for exponential moving average over the codebooks. epsilon (float): Epsilon value for numerical stability. kmeans_init (bool): Whether to use kmeans to initialize the codebooks. kmeans_iters (int): Number of iterations used for kmeans initialization. threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes that have an exponential moving average cluster size less than the specified threshold with randomly selected vector from the current batch. commitment_weight (float): Weight for commitment loss. NrHrTé2rçð?r2rIÚ codebook_dimrrrJrKrLÚcommitment_weightc s~tƒ ¡t||ƒ} | |k} | rt || ¡nt ¡|_| r#t | |¡nt ¡|_||_| |_ t | ||||||d|_ ||_ dS)N)r2rIrJrKrrrL) rQrRr rÚLinearÚIdentityÚ project_inÚ project_outrr‹rGÚ _codebookrI) rWr2rIrŠrrrJrKrLr‹Ú _codebook_dimÚrequires_projectionrYr r rRùs ý zVectorQuantization.__init__cCs|jjSr )rrO)rWr r r ÚcodebookszVectorQuantization.codebookcCs$t|dƒ}| |¡}|j |¡}|S)Núb d n -> b n d)rrŽrrt)rWrÚembed_inr r r rts   zVectorQuantization.encodecCs$|j |¡}| |¡}t|dƒ}|S)Núb n d -> b d n)rrurrrrr r r rus   zVectorQuantization.decodecCsž|j}t|dƒ}| |¡}| |¡\}}|jr||| ¡}tjdg||jd}|jr@|jdkr@t   | ¡|¡}|||j}|  |¡}t|dƒ}|||fS)Nr”ç)r&Ú requires_gradrr–) r&rrŽrrxÚdetachrÚtensorr‹rpÚmse_lossr)rWrr&rmrlÚlossÚ commit_lossr r r r}%s      zVectorQuantization.forward)NrHrTrˆrr‰)r~rr€rr‚ÚtpÚOptionalrƒÚboolrRÚpropertyr“rtrur}r†r r rYr r‡ésBöþýüûúùø ÷ ö r‡csneZdZdZ‡fdd„Zddejefdd„Zdde j dejed e j fd d „Z d e j d e j fd d„Z ‡Z S)ÚResidualVectorQuantizationzrResidual vector quantization implementation. Follows Algorithm 1. in https://arxiv.org/pdf/2107.03312.pdf c s,tƒ ¡t ‡fdd„t|ƒDƒ¡|_dS)Ncsg|] }tdiˆ¤Ž‘qS)r )r‡)Ú.0r>©Úkwargsr r Ú Bsz7ResidualVectorQuantization.__init__..)rQrRrÚ ModuleListr6Úlayers)rWÚnum_quantizersr¥rYr¤r rR?s  ÿz#ResidualVectorQuantization.__init__NÚn_qc Cs‚d}|}g}g}|pt|jƒ}|jd|…D]}||ƒ\}} } ||}||}| | ¡| | ¡qttj||fƒ\} } || | fS)Nr—)Úlenr¨ÚappendÚmaprÚstack) rWrrªÚ quantized_outÚresidualÚ all_lossesÚ all_indicesÚlayerÚ quantizedr+rœÚ out_lossesÚ out_indicesr r r r}Es   z"ResidualVectorQuantization.forwardrr c Cs^|}g}|p t|jƒ}|jd|…D]}| |¡}| |¡}||}| |¡qt |¡}|Sr )r«r¨rtrur¬rr®) rWrrªr°r²r³r+r´r¶r r r rtYs    z!ResidualVectorQuantization.encodeÚ q_indicescCsBtjd|jd}t|ƒD]\}}|j|}| |¡}||}q |S)Nr—r%)rršr&Ú enumerater¨ru)rWr·r¯Úir+r³r´r r r rues    z!ResidualVectorQuantization.decoder )r~rr€rrRržrŸr‚r}rrUrtrur†r r rYr r¢;s  " r¢)r)r-)rÚtypingržÚeinopsrrrrÚtorch.nn.functionalÚ functionalrpÚrÚAnyr rƒrr‚rr#r,rFÚModulerGr‡r¢r r r r Ús"   R