o mi4@sdZddlZddlmZmZddlZddlmZddlmm Z ddl m Z m Z dejdejd ejfd d Zd efd dZd%dedefddZdefddZdefddZd&dedefddZGdd d ejZGd!d"d"ejZGd#d$d$ejZdS)'z(Core vector quantization implementation.N) rearrangerepeat)nn)broadcast_tensorsrankvaldreturncCs|dur|S|SN)rr r r D/home/ubuntu/SpeechTokenizer/speechtokenizer/quantization/core_vq.pydefault+srdecaycCs|j|j|d|ddS)Nr)alpha)datamul_add_) moving_avgnewrr r r ema_inplace/srh㈵> n_categoriesepsiloncCs|||||Sr )sum)xrrr r r laplace_smoothing3srshapecGst|}tj||Sr )torchemptyrinitkaiming_uniform_)rtr r r uniform_init7s  r#numcCsT|jd|j}}||krtj||dd|}||Stjd||f|d}||S)Nrdevice)rr&rrandpermrandint)samplesr$ num_samplesr&indicesr r r sample_vectors=s r, num_clusters num_itersc Cs|jd|j}}t||}t|D]P}t|dt|d}|djdd }|jddj} tj | |d} | dk} | | d} | j |||d } | dt | d |d || | d } t| d || }q|| fS) Nz n d -> n () dz c d -> () c ddim) minlengthrr)dtypezn -> n d)r .N)rr5r,rangerrmaxr+rbincount masked_fill new_zeros scatter_add_rwhere)r)r.r/r3r5means_diffsdistsbucketsbins zero_maskbins_min_clamped new_meansr r r kmeansHs      rGcseZdZdZ     d$deded ed ed ed ed effdd Zejj ddZ ddZ ddZ ddZ ddZddZddZddZd d!Zd"d#ZZS)%EuclideanCodebookaJCodebook with Euclidean distance. Args: dim (int): Dimension. codebook_size (int): Codebook size. kmeans_init (bool): Whether to use k-means to initialize the codebooks. If set to true, run the k-means algorithm on the first training batch and use the learned centroids as initialization. kmeans_iters (int): Number of iterations used for k-means algorithm at initialization. decay (float): Decay for exponential moving average over the codebooks. epsilon (float): Epsilon value for numerical stability. threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes that have an exponential moving average cluster size less than the specified threshold with randomly selected vector from the current batch. Fr-Gz?rr1r3 codebook_size kmeans_init kmeans_itersrrthreshold_ema_dead_codec st||_|s tntj}|||} ||_||_||_||_ | dt | g| dt|| d| | d| dS)Ninited cluster_sizeembed embed_avg) super__init__rr#rzerosrJrLrrMregister_bufferTensorclone) selfr3rJrKrLrrrMinit_fnrP __class__r r rSps   zEuclideanCodebook.__init__cCsf|jrdSt||j|j\}}|jj||jj||j j||jjt dgdS)NT) rNrGrJrLrPrcopy_rQrWrOrrV)rXrrPrOr r r init_embed_szEuclideanCodebook.init_embed_cCs.t|dt||j|j}|jj|dS)Nr6)rr=r,rJrPrr\)rXr)maskmodified_codebookr r r replace_szEuclideanCodebook.replace_cCsD|jdkrdS|j|jk}t|sdSt|d}|j||ddS)Nr... d -> (...) d)r^)rMrOranyrr`)rX batch_samples expired_codesr r r expire_codes_s    zEuclideanCodebook.expire_codes_cCst|d}|S)Nra)r)rXrr r r preprocess zEuclideanCodebook.preprocesscCsR|j}|djdddd|||djddd }|jddj}|S)Nr1rT)keepdimrr0r2)rPr"powrr8r+)rXrrPdist embed_indr r r quantizes  zEuclideanCodebook.quantizecCs|j|ddS)Nr0)view)rXrkrr r r postprocess_embsz!EuclideanCodebook.postprocess_embcCst||j}|Sr )F embeddingrPrXrkrlr r r dequantizeszEuclideanCodebook.dequantizecCs*|j}||}||}|||}|Sr )rrfrlrn)rXrrrkr r r encodes    zEuclideanCodebook.encodecCs||}|Sr )rrrqr r r decodergzEuclideanCodebook.decodec Cs|j|j}}||}||||}t||j|}| ||}| |}|j rk| |t |j|d|j||}t |j||jt|j|j|j|j}|j|d} |jj| ||fS)Nrr)rr5rfr]rlroone_hotrJtypernrrtrainingrerrOrrr"rQrr unsqueezerPrr\) rXrrr5rk embed_onehotrl embed_sumrOembed_normalizedr r r forwards&       zEuclideanCodebook.forward)Fr-rIrr1)__name__ __module__ __qualname____doc__intfloatrSrjitignorer]r`rerfrlrnrrrsrtr| __classcell__r r rZr rHasB     rHcseZdZdZ       dd ed ed ejed ed ededededeffdd Z e ddZ ddZ ddZ ddZZS)VectorQuantizationaVector quantization implementation. Currently supports only euclidean distance. Args: dim (int): Dimension codebook_size (int): Codebook size codebook_dim (int): Codebook dimension. If not defined, uses the specified dimension in dim. decay (float): Decay for exponential moving average over the codebooks. epsilon (float): Epsilon value for numerical stability. kmeans_init (bool): Whether to use kmeans to initialize the codebooks. kmeans_iters (int): Number of iterations used for kmeans initialization. threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes that have an exponential moving average cluster size less than the specified threshold with randomly selected vector from the current batch. commitment_weight (float): Weight for commitment loss. NrIrT2r1?r3rJ codebook_dimrrrKrLrMcommitment_weightc s~tt||} | |k} | rt|| nt|_| r#t| |nt|_||_| |_ t | ||||||d|_ ||_ dS)N)r3rJrKrLrrrM) rRrSrrLinearIdentity project_in project_outrrrH _codebookrJ) rXr3rJrrrrKrLrMr _codebook_dimrequires_projectionrZr r rSs  zVectorQuantization.__init__cCs|jjSr )rrP)rXr r r codebookszVectorQuantization.codebookcCs$t|d}||}|j|}|S)Nb d n -> b n d)rrrrs)rXrembed_inr r r rss   zVectorQuantization.encodecCs$|j|}||}t|d}|S)Nb n d -> b d n)rrtrrrqr r r rts   zVectorQuantization.decodecCs|j}t|d}||}||\}}|jr|||}tjdg||jd}|jr@|jdkr@t ||}|||j}| |}t|d}|||fS)Nr)r& requires_gradrr) r&rrrrwdetachrtensorrromse_lossr)rXrr&rlrkloss commit_lossr r r r|$s      zVectorQuantization.forward)NrIrTrr1r)r}r~rrrtpOptionalrboolrSpropertyrrsrtr|rr r rZr rsB   rc seZdZdZfddZddejedejefddZ dd e j dejed ejed e j fd d Z dde j d ed e j fddZ ZS)ResidualVectorQuantizationzrResidual vector quantization implementation. Follows Algorithm 1. in https://arxiv.org/pdf/2107.03312.pdf c s,ttfddt|D|_dS)Ncsg|] }tdiqS)r )r).0r?kwargsr r Asz7ResidualVectorQuantization.__init__..)rRrSr ModuleListr7layers)rXnum_quantizersrrZrr rS>s  z#ResidualVectorQuantization.__init__Nn_qrcCsd}|}g}g}g}|pt|j}t|jd|D](\} } | |\} } } || }|| }|| || |rB| |vrB|| qttj||f\}}||||fS)Nr)lenr enumerateappendmaprstack)rXrrr quantized_outresidual all_losses all_indices out_quantizedilayer quantizedr+r out_losses out_indicesr r r r|Ds"     z"ResidualVectorQuantization.forwardrstr c Csf|}g}|p t|j}|pd}|j||D]}||}||}||}||qt|} | S)Nr)rrrsrtrrr) rXrrrrrrr+rrr r r rs[s    z!ResidualVectorQuantization.encoder q_indicescCsFtjd|jd}t|D]\}}|j||}||}||}q |S)Nrr%)rrr&rrrt)rXrrrrr+rrr r r rths   z!ResidualVectorQuantization.decode)NN)r)r}r~rrrSrrrlistr|rrVrsrtrr r rZr r:s   ,$ r)r)r-)rtypingreinopsrrrrtorch.nn.functional functionalrodistribrrAnyrrrrrr#r,rGModulerHrrr r r r s"  R