o
    ̳im>                     @  s"  d dl mZ d dlmZ d dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlZd dlmZmZ d dlmZmZ d dlm  mZ d dlmZ d d	lmZ d d
lmZmZmZmZmZ d dl m!Z! dd Z"dd Z#dd Z$dd Z%e	dd Z&G dd deZ'G dd deZ(G dd deZ)dS )    )annotations)ListNceil)partialcache)zip_longest)nnTensor)Module
ModuleListVectorQuantize)	rearrangerepeatreducepackunpack)get_atc                 C  s   | d uS N )valr   r   W/home/ubuntu/.local/lib/python3.10/site-packages/vector_quantize_pytorch/residual_vq.pyexists      r   c                 C  s   | d S Nr   r   )itr   r   r   first   r   r   c                 C  s   t | r| S |S r   )r   )r   dr   r   r   default      r   c                 C  s   t | | | S r   r   )nummultr   r   r   round_up_multiple   r    r#   c                   C  s   t  o	t  dkS )N   )distis_initializedget_world_sizer   r   r   r   is_distributed$   s   r(   c                      s,   e Zd Z			d fdd	Zdd Z  ZS )	MLPN   Fc                   st   t    t||}td| || _tg }t|D ]}|t	t||t
 t|| q|| _|| _d S )N   )super__init__r   r	   Linearproj_inr   rangeappend
SequentialSiLUlayersl2norm_output)selfdim
dim_hiddendepthr5   r4   _	__class__r   r   r-   ,   s   




zMLP.__init__c          
      C  s   |j dk}|rt|d}|jd |jd |jd |jd f\}}}}t|d||d}t|d||d}tj||fd	d
}| |}| jD ]}	|	|| }qC| jrVt	j
|d	d
}|sZ|S t|dS )Nr+   zc d -> 1 c dr   zh c d -> h b n c d)nbzb n d -> h b n c d)chr7   z1 ... -> ...)ndimr   shaper   torchcatr/   r4   r5   F	normalize)
r6   codes	condition
one_headedheads	num_codesbatchseq_lenxlayerr   r   r   forwardD   s   

,


zMLP.forward)Nr*   F)__name__
__module____qualname__r-   rS   __classcell__r   r   r;   r   r)   +   s    r)   c                
      s   e Zd ZdZdddddddde d	d fd	d
Zedd Zedd Zedd Z	dd Z
dd Z						ddddZ  ZS )
ResidualVQz> Follows Algorithm 1. in https://arxiv.org/pdf/2107.03312.pdf NFr$   r   )	codebook_dimshared_codebookrM   quantize_dropoutquantize_dropout_cutoff_indexquantize_dropout_multiple_ofaccept_image_fmapimplicit_neural_codebook
mlp_kwargsr`   dictc                  sb  t    |dksJ dt|| }||k}|r"t||nt _|r/t||nt _|_|_	 _
|
_|
rIjddd |rRjddd t fddt|D _td	d jD snJ |os|dk_|d
ks{J |_|_d _|
rtfddt|d D _|_|sd S j^}}|j}|D ]}||_qd S )Nr$   z5residual vq is not compatible with multi-headed codesTF)learnable_codebook
ema_update)manual_ema_update manual_in_place_optimizer_updatec                   s"   g | ]}t d d qS ))r7   rY   r^   r   r   .0r:   )r^   rY   	vq_kwargsr   r   
<listcomp>   s   " z'ResidualVQ.__init__.<locals>.<listcomp>c                 S  s   g | ]}|j  qS r   )has_projections)rg   vqr   r   r   ri          r   c                   s(   g | ]}t d tjjd qS ))r7   r5   r   )r)   r   r4   use_cosine_simrf   )rY   r`   r6   r   r   ri      s   ( )r,   r-   r   r	   r.   Identity
project_inproject_outrj   num_quantizersr^   r_   updater   r0   r4   allr[   r\   r]   mlpsrZ   	_codebook)r6   r7   rq   rY   rZ   rM   r[   r\   r]   r^   r_   r`   rh   codebook_input_dimrequires_projectionfirst_vqrest_vqcodebookrk   r;   )r^   rY   r`   r6   rh   r   r-   f   sL   

 $
zResidualVQ.__init__c                 C     | j d jS r   )r4   codebook_sizer6   r   r   r   r|         zResidualVQ.codebook_sizec                 C  r{   r   )r4   rY   r}   r   r   r   rY      r~   zResidualVQ.codebook_dimc                 C  s,   dd | j D }tj|dd}t|d}|S )Nc                 S  s   g | ]}|j jqS r   )ru   embed)rg   rR   r   r   r   ri      rl   z(ResidualVQ.codebooks.<locals>.<listcomp>r   rC   zq 1 c d -> q c d)r4   rF   stackr   )r6   	codebooksr   r   r   r      s   
zResidualVQ.codebooksc                 C  s,  |j d |j d }}t|gd\}}|| jk r.| jdks!J dtj|d| j| fdd}|dk}||d}| jsCtd| j	|}nAd g| j
R }g }d}t| j	|jdd	|D ]%\}	}}
t|
ro|
|	|d
}	td|	|}ntd|	|}|| ||7 }qYt|}|t|dd}t||d\}|S )Nr   rB   zb * q        zmquantize dropout must be greater than 0 if you wish to reconstruct from a signal with less fine quantizations)value      zq [c] d, b n q -> q b n drC   rK   zb n [c] d, b n -> b n dz[c] d, b n -> b n dzb n q -> q b n 1zq b * d)rE   r   rq   r[   rH   padmasked_fillr_   r   r   rt   zipunbindr   r1   rF   r   r   r   )r6   indicesrO   quantize_dimpsmask	all_codescode_transform_mlpsquantized_outrJ   maybe_transform_mlplayer_codesr   r   r   get_codes_from_indices   s.   
 


z!ResidualVQ.get_codes_from_indicesc                 C  s    |  |}t|dd}| |S )Nzq ... -> ...sum)r   r   rp   )r6   r   rJ   codes_summedr   r   r   get_output_from_indices   s   

z"ResidualVQ.get_output_from_indicesr   Tensor | List[Tensor] | Nonec           '   	   C  s   | j | jt||jf\}}	}
}| |}| jrt|rJ d}|}g }g }t|tr/t	|}|
r>t
|dkr<J dg }| joF| joF|
 }|rt|rSt|}nt rmtjtd|d}t| }t|}nt}|| j|}|	dkrt|d |	d }| jr|jd g|jdd  R nt|jd d	 }tj|d
|tjd}tjdd||jd}dt| j }| jrd g| jR }g }t t!| j|D ]a\}\}}|r||kr|"| |"| qd }|
r|d|f }t|rt#||d}|"| |||||||d^}} ||$  }|| }|
r| d }!|"|! q| \}"}#|"|" |"|# q| jrN| j%rNt&| j}$|$j'(  |$)  |$*tj+|dd | ,|}|
r\|t-|fS t.t#tj	dd||f\}}|||f}%|r~| /|}&g |%|&R }%|%S )Nr   rB   zsome of the residual vq indices were dropped out. please use indices derived when the module is in eval mode to derive cross entropy lossi'  )devicer$   r   r=   r+   r   )r   dtype)r$   r   .r   )r   r   sample_codebook_tempfreeze_codebookcodebook_transform_fnrC   )0rq   r]   r   r   ro   r^   
isinstancelistrF   r   anytrainingr[   randomRandomr(   tensor	randranger%   
all_reduceitemr\   r#   rE   tuplefulllongr   lenr4   r_   rt   	enumerater   r1   r   detachrZ   r   ru   
update_emaupdate_in_place_optimizerexpire_codes_rG   rp   r   mapr   )'r6   rQ   r   r   return_all_codesr   r    rand_quantize_dropout_fixed_seed	num_quantquant_dropout_multiple_ofreturn_lossr   r   residual
all_lossesall_indices	ce_lossesshould_quantize_dropoutrandtdropout_seedrand_quantize_dropout_indexnull_indices_shapenull_indices	null_lossmaybe_code_transformsall_residualsquantizer_indexrk   	maybe_mlplayer_indices	quantizedrestce_lossembed_indiceslossshared_layerretr   r   r   r   rS      s   



4



	






zResidualVQ.forward)r`   ra   )NNFNFN)r   r   )rT   rU   rV   __doc__ra   r-   propertyr|   rY   r   r   r   rS   rW   r   r   r;   r   rX   d   s6    K


8rX   c                      s`   e Zd Zddd fdd
Zedd Zedd	 Zd
d Zdd Z					dddZ	  Z
S )GroupedResidualVQr$   F)groupsr^   c                  sl   t    || _|| _|| dksJ || }|| _tg | _t|D ]}| jt	d||d| q#d S )Nr   )r7   r^   r   )
r,   r-   r7   r   r^   r   rvqsr0   r1   rX   )r6   r7   r   r^   kwargsdim_per_groupr:   r;   r   r   r-     s   



zGroupedResidualVQ.__init__c                 C  s   t tdd | jD S )Nc                 s  s    | ]}|j V  qd S r   )r   )rg   rvqr   r   r   	<genexpr>  s    z.GroupedResidualVQ.codebooks.<locals>.<genexpr>)rF   r   r   r   r}   r   r   r   r     s   zGroupedResidualVQ.codebooksc                 C  s   | j rdS dS )Nr$   rB   )r^   r}   r   r   r   	split_dim  s   zGroupedResidualVQ.split_dimc                 C  s$   t dd t| j|D }t|S )Nc                 s      | ]
\}}| |V  qd S r   )r   rg   r   chunk_indicesr   r   r   r         z;GroupedResidualVQ.get_codes_from_indices.<locals>.<genexpr>)r   r   r   rF   r   )r6   r   rJ   r   r   r   r     s   
z(GroupedResidualVQ.get_codes_from_indicesc                 C  s*   t dd t| j|D }tj|| jdS )Nc                 s  r   r   )r   r   r   r   r   r     r   z<GroupedResidualVQ.get_output_from_indices.<locals>.<genexpr>rC   )r   r   r   rF   rG   r   )r6   r   outputsr   r   r   r     s   z)GroupedResidualVQ.get_output_from_indicesNc              
     s  |j | j}}|| | jksJ |j| j|d}t|t }t|dk}	t|dks3t|| jks3J t||||t	
dtdd t fddt| j||D }
tt|
 }
|	ri|
\}}tj||dt|fS |
^}}}}tj||d}t|}t|}|||g|R }|S )NrC   r   g    cA)r   r   r   r   r   c                 3  s*    | ]\}}}||fd |i V  qdS )r   Nr   )rg   r   chunkr   forward_kwargsr   r   r     s   ( z,GroupedResidualVQ.forward.<locals>.<genexpr>)rE   r   r7   r   r   r   r   r   ra   r   randintintr   r   r   rF   rG   r   r   )r6   rQ   r   r   r   r   r   rE   r   return_ce_lossoutr   r   r   commit_lossesmaybe_all_codesr   r   r   r   rS     s0   	 


zGroupedResidualVQ.forward)NFNFN)rT   rU   rV   r-   r   r   r   r   r   rS   rW   r   r   r;   r   r     s     

r   )*
__future__r   typingr   r   mathr   	functoolsr   r   	itertoolsr   rF   r	   r
   torch.nnr   r   torch.nn.functional
functionalrH   torch.distributeddistributedr%   /vector_quantize_pytorch.vector_quantize_pytorchr   einopsr   r   r   r   r   einxr   r   r   r   r#   r(   r)   rX   r   r   r   r   r   <module>   s2    
9  0