o
    ̳i!                     @   s   d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlmZm	Z	 d dl
m  mZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZ d
d Zdd Zdd ZG dd deZG dd deZdS )    N)log2)partial)nn)Module
ModuleList)autocast)LFQ)	rearrangerepeatreducepackunpack)get_atc                 C   s   | d uS N )valr   r   X/home/ubuntu/.local/lib/python3.10/site-packages/vector_quantize_pytorch/residual_lfq.pyexists   s   r   c                 C   s   t | r| S |S r   )r   )r   dr   r   r   default      r   c                 C   s   t | | | S r   )ceil)nummultr   r   r   round_up_multiple   r   r   c                       sX   e Zd ZdZddddd fdd
Zed	d
 Zdd Zdd Z			dddZ	  Z
S )ResidualLFQz> Follows Algorithm 1. in https://arxiv.org/pdf/2107.03312.pdf Fr      N)quantize_dropoutquantize_dropout_cutoff_indexquantize_dropout_multiple_ofsoft_clamp_input_valuec                   s   t    tt|}	|	|k}
|
rt||	nt | _|
r$t|	|nt | _|
| _	|| _
tg | _t|D ] }d|  }td|	||d|}| j| t|rY|d9 }q9tdd | jD sfJ |ok|dk| _|dkssJ || _|| _d S )	N   )dimcodebook_scaler    g      ?c                 S   s   g | ]}|j  qS r   )has_projections).0lfqr   r   r   
<listcomp>H   s    z(ResidualLFQ.__init__.<locals>.<listcomp>r   r   r   )super__init__intr   r   LinearIdentity
project_inproject_outr$   num_quantizersr   layersranger   appendr   allr   r   r   )selfr"   r/   codebook_sizer   r   r   r    kwargscodebook_dimrequires_projectionindr#   r&   	__class__r   r   r)   !   s4   


zResidualLFQ.__init__c                 C   s"   dd | j D }tj|dd}|S )Nc                 S   s   g | ]}|j qS r   )codebook)r%   layerr   r   r   r'   S   s    z)ResidualLFQ.codebooks.<locals>.<listcomp>r   r"   )r0   torchstack)r4   	codebooksr   r   r   rA   Q   s   zResidualLFQ.codebooksc                 C   s   |j d |j d }}t|gd\}}|| jk r.| jdks!J dtj|d| j| fdd}|dk}||d}td| j|}|t	|d	d}t
||d
\}|S )Nr   zb * q        zmquantize dropout must be greater than 0 if you wish to reconstruct from a signal with less fine quantizations)value      zq [c] d, b n q -> q b n dzb n q -> q b n 1zq b * d)shaper   r/   r   Fpadmasked_fillr   rA   r	   r   )r4   indicesbatchquantize_dimpsmask	all_codesr   r   r   get_codes_from_indicesW   s   
z"ResidualLFQ.get_codes_from_indicesc                 C   s    |  |}t|dd}| |S )Nzq ... -> ...sum)rP   r   r.   )r4   rJ   codescodes_summedr   r   r   get_output_from_indicesw   s   

z#ResidualLFQ.get_output_from_indicesc                 C   s  | j | j|j}}}| |}d}|}	g }
g }| jo| j}|rWt|r)t|nt}|	| j
|}|dkr?t|d |d }tj|jd d d|tjd}tjd||jd}tddd@ t| jD ]2\}}|rx||krx|| |
| qc||	|d	\}}}|	|  }	|| }|| |
| qcW d    n1 sw   Y  | |}tttjd
d|
|f\}
}|||
f}|s|S | |}g ||R S )NrC   r   r!   rE   )devicedtypecudaF)enabled)rN   rB   r>   )r/   r   rU   r-   trainingr   r   randomRandom	randranger   r   r?   fullrF   longtensorrV   r   	enumerater0   r2   detachr.   mapr   r@   rP   )r4   xrN   return_all_codes rand_quantize_dropout_fixed_seed	num_quantquant_dropout_multiple_ofrU   quantized_outresidual
all_lossesall_indicesshould_quantize_dropoutrandrand_quantize_dropout_indexnull_indices	null_lossquantizer_indexr=   	quantizedrJ   lossretrO   r   r   r   forward|   sD   






zResidualLFQ.forward)NFN)__name__
__module____qualname____doc__r)   propertyrA   rP   rT   ru   __classcell__r   r   r:   r   r      s    0
 r   c                       sZ   e Zd Zddd fdd
Zedd Zedd	 Zd
d Zdd Z		dddZ	  Z
S )GroupedResidualLFQr   F)groupsaccept_image_fmapc                   sl   t    || _|| _|| dksJ || }|| _tg | _t|D ]}| j	t
dd|i| q$d S )Nr   r"   r   )r(   r)   r"   r}   r~   r   r   rvqsr1   r2   r   )r4   r"   r}   r~   r6   dim_per_group_r:   r   r   r)      s   

zGroupedResidualLFQ.__init__c                 C   s   t tdd | jD S )Nc                 s   s    | ]}|j V  qd S r   )rA   )r%   rvqr   r   r   	<genexpr>   s    z/GroupedResidualLFQ.codebooks.<locals>.<genexpr>)r?   r@   tupler   r4   r   r   r   rA      s   zGroupedResidualLFQ.codebooksc                 C   s   | j rdS dS )Nr   rB   )r~   r   r   r   r   	split_dim   s   zGroupedResidualLFQ.split_dimc                 C   s$   t dd t| j|D }t|S )Nc                 s       | ]
\}}| |V  qd S r   )rP   r%   r   chunk_indicesr   r   r   r          z<GroupedResidualLFQ.get_codes_from_indices.<locals>.<genexpr>)r   zipr   r?   r@   )r4   rJ   rR   r   r   r   rP      s   
z)GroupedResidualLFQ.get_codes_from_indicesc                 C   s*   t dd t| j|D }tj|| jdS )Nc                 s   r   r   )rT   r   r   r   r   r      r   z=GroupedResidualLFQ.get_output_from_indices.<locals>.<genexpr>r>   )r   r   r   r?   catr   )r4   rJ   outputsr   r   r   rT      s   z*GroupedResidualLFQ.get_output_from_indicesNc                    s   |j | j}}|| | jksJ |j| j|d}t||tdtdd t	 fddt
| j|D }t	t
| }|^}}}	}
tj||d}t|}t|	}	|||	g|
R }|S )Nr>   r   g    cA)rN   rd   re   c                 3   s$    | ]\}}||fi  V  qd S r   r   )r%   r   chunkforward_kwargsr   r   r     s   " z-GroupedResidualLFQ.forward.<locals>.<genexpr>)rF   r   r"   r   r}   dictrZ   randintr*   r   r   r   r?   r   r@   )r4   rc   rN   rd   rF   r   outrr   rk   commit_lossesmaybe_all_codesrt   r   r   r   ru      s    

zGroupedResidualLFQ.forward)NF)rv   rw   rx   r)   rz   rA   r   rP   rT   ru   r{   r   r   r:   r   r|      s    

r|   )rZ   mathr   	functoolsr   r?   r   torch.nnr   r   torch.nn.functional
functionalrG   	torch.ampr   0vector_quantize_pytorch.lookup_free_quantizationr   einopsr	   r
   r   r   r   einxr   r   r   r   r   r|   r   r   r   r   <module>   s"     )