o
    i-                     @   s  d dl Z d dl mZ d dl mZ d dlmZmZmZ ddlmZm	Z	m
Z
mZmZ ddlmZ G dd	 d	ejZG d
d dejZG dd dejZG dd dejZG dd dejZG dd dejZddddZG dd dejZdd ZeZG dd dejZG dd dejZdS )    N)Tensor)nn)DynamicPositionBiasLocalMHAFeedForward   )ChannelNormConv1dLinearGRNSnake1d)
FirstBlockc                       sN   e Zd Z											d fd	d
	ZdddZe	dddZ  ZS )
LocalTrans      T@                 Fc                    s   t    tg | _|| _|
 | _| jrd nt|d |d| _t	|D ]"}| j
tt|||||| jdd | jd|ddt|||	dg q$d S )N   )dimheadsFT)r   dim_headr   dropoutcausalwindow_sizeuse_xposxpos_scale_baseuse_rotary_pos_embprenorm
qk_rmsnormexact_windowsize)r   multr   )super__init__r   
ModuleListlayersr   r   r   dynamic_pos_biasrangeappendr   r   )selfr   depthr   local_attn_window_sizer   r   ff_multattn_dropout
ff_dropoutuse_dynamic_pos_biasr    _	__class__ R/home/ubuntu/.local/lib/python3.10/site-packages/neucodec/codec_encoder_distill.pyr$   
   s<   
zLocalTrans.__init__Nc                 C   sP   | j rd n	| | j| jd }| jD ]\}}||||d| }||| }q|S )Nr   )mask	attn_bias)r   r'   r   r&   )r*   xr6   r7   attnffr4   r4   r5   forward;   s   zLocalTrans.forward   r      c              
   C   s   | |||d ddd||dS )Nr   r   T)r   r+   r   r   r-   r   r,   r0   r4   )clsfeature_dimr+   local_window_sizer0   r4   r4   r5   builderG   s   zLocalTrans.builder)r   r   Tr   r   r   r   r   r   FFNr<   r   r=   F)__name__
__module____qualname__r$   r;   classmethodrA   __classcell__r4   r4   r2   r5   r   	   s"    
1r   c                       s.   e Zd Z				d	 fdd	Zdd Z  ZS )
LocalEncoderr<   r   r=   Fc                    s"   t    tj||||d| _d S )N)r?   r+   r@   r0   )r#   r$   r   rA   local_trans)r*   r?   r+   r@   r0   r2   r4   r5   r$   X   s   
zLocalEncoder.__init__c                 C   s   | ddd}| |}|S )zl
        Args:
            feature: (B, C, T)
        Returns:
            local_feature: (B, T, C)
        r   r   r   )permuterJ   )r*   featurer4   r4   r5   r;   g   s   
zLocalEncoder.forwardrC   rD   rE   rF   r$   r;   rH   r4   r4   r2   r5   rI   W   s    rI   c                       s(   e Zd Z	d fdd	Zdd Z  ZS )		DownTransr<   r=   r   c                    sV   t    || dksJ || _|| _tj|f||d|| _t||||d| _d S )Nr   r@   r+   kernel_sizestride)	r#   r$   r?   compress_rater   rA   transr	   
down_layer)r*   r?   r   rS   r+   kwargsr2   r4   r5   r$   t   s   
zDownTrans.__init__c                 C   s,   |  |}| |dddddd}|S Nr   r   r   )rT   rU   rK   r*   r8   r4   r4   r5   r;      s   
zDownTrans.forward)r<   r=   r   r   rM   r4   r4   r2   r5   rN   s   s    rN   c                       s0   e Zd Z					d
 fdd	Zdd	 Z  ZS )CompressedLocalEncoderWithCacher<   r=   r      r   c                    s   t    || _|| _|| _|| | _tt	d| j| j || _
t|f| j| |dd|| _tj|f| j|d d|| _d S )Nr   r   )r   rS   r+   rO   )r#   r$   r@   
cache_sizerS   trans_window_sizer   	Parametertorchrandncache_tokenrN   
down_transr   rA   rJ   )r*   r?   r@   rS   r[   r+   rV   r2   r4   r5   r$      s2   
	
z(CompressedLocalEncoderWithCache.__init__c                    sp   | ddd}tj|| j| j dd}| j|jd dd tj fdd|D dd}| 	|}| 
|}|S )Nr   r   r   )r   c                    s   g | ]} |fD ]}|qqS r4   r4   ).0fsfr`   r4   r5   
<listcomp>   s    z;CompressedLocalEncoderWithCache.forward.<locals>.<listcomp>)rK   r^   splitr@   rS   r`   expandshapecatra   rJ   )r*   rL   split_featurer4   rf   r5   r;      s   


z'CompressedLocalEncoderWithCache.forward)r<   r=   r   rZ   r   rM   r4   r4   r2   r5   rY      s    "rY   c                       s*   e Zd ZdZd
 fdd	Zdd	 Z  ZS )ConvUnitz<
    Args:
        dim (int): Number of input channels.
    TFr      c                    s   t    |d | }t|||||d |d| _|r t|ddnt | _t|d| | _	|r9t
d| dd| _nt | _td| | _td| || _d S )Nr   r   )rQ   dilationpaddinggroupschannels_lastdata_formatr   )r#   r$   r	   dw_convr   r   Identitynormr
   pw_conv1r   actGELUr   grnpw_conv2r*   r   	snake_actrw   ro   rQ   	total_padr2   r4   r5   r$      s(   


zConvUnit.__init__c                 C   s\   |  |}|ddd}| |}| |}| |}| |}| |}|ddd}|S rW   )ru   rK   rw   rx   ry   r{   r|   rX   r4   r4   r5   r;      s   





zConvUnit.forwardTFr   rn   )rD   rE   rF   __doc__r$   r;   rH   r4   r4   r2   r5   rm      s    rm   c                       sL   e Zd Z	ddejdedef fddZdefd	d
Z	defddZ
  ZS )Residualr   Tmodule	drop_probscale_by_keepc                    s<   t    d|  krdk sJ  J || _|| _|| _d S )Nr   r   )r#   r$   r   r   r   )r*   r   r   r   r2   r4   r5   r$      s
   

zResidual.__init__x_sidec                 C   s`   | j dks| js
|S d| j  }|jd fd|jd   }|||}| jr,|| || S )Nr   r   r   )r   )r   trainingrj   ndim	new_empty
bernoulli_r   div_)r*   r   	keep_probrj   	keep_maskr4   r4   r5   	drop_path   s   

zResidual.drop_pathr8   c                 C   s   |  |}| |}|| S rB   )r   r   )r*   r8   r   r4   r4   r5   r;     s   

zResidual.forward)r   T)rD   rE   rF   r   Modulefloatboolr$   r   r   r;   rH   r4   r4   r2   r5   r      s    	
r   r   )	drop_ratec                 O   s   t t|i || dS )Nr   )r   rm   )r   argsrV   r4   r4   r5   <lambda>
      r   c                       s&   e Zd Zd	 fdd	Zdd Z  ZS )

LegacyUnitTFr   rn   c              
      sl   t    |sJ d|dksJ d|d | }tt|t|||||d dt|t||dd| _d S )Nz'LegacyUnit only supports snake_act=TrueFz#LegacyUnit only supports norm=Falser   r   )rQ   ro   rp   )rQ   )r#   r$   r   
Sequentialr   r	   blockr}   r2   r4   r5   r$     s    

zLegacyUnit.__init__c                 C   
   |  |S rB   )r   rX   r4   r4   r5   r;   "     
zLegacyUnit.forwardr   rM   r4   r4   r2   r5   r     s    r   c                  O   s   t t| i |ddS )Nr   r   )r   r   )r   rV   r4   r4   r5   r   &  r   c                       sJ   e Zd Z							dded	ed
ededef
 fddZdd Z  ZS )Encoderr   r   r   r   r   r   r   r   r   r       r   r<      r   r   FTr?   stridesdepthsdimsdrop_path_ratec              	      s  t    td g}dd td|t|D d td d dd  ||D ]8\}	}
}tj fddt	|D  }tt
|	|
|
drTt|	dd	nt }|||g7 } |7  q+|tj fd
dt	|d D  t
d |dddg7 }tj| | _d S )Nr   c                 S   s   g | ]}|  qS r4   )item)rc   r8   r4   r4   r5   rg   ?  s    z$Encoder.__init__.<locals>.<listcomp>rb   r   c                    s$   g | ]}t  |  d qS )r   r   r~   rw   BaseUnitrc   j)curdrop_path_ratesi_duse_normuse_snake_actr4   r5   rg   E  s    
rP   channels_firstrs   c                    s(   g | ]}t d   |  dqS )rb   r   r   r   )r   r   r   r   r   r4   r5   rg   [  s    
rZ   )rQ   rp   )r#   r$   r   r^   linspacesumzipr   r   r(   r	   r   rv   blocks)r*   r?   r   r   r   r   r   r   r   o_drR   r+   stagerU   r2   )r   r   r   r   r   r   r5   r$   .  s<   


*

zEncoder.__init__c                 C   r   rB   )r   rX   r4   r4   r5   r;   k  r   zEncoder.forward)r   r   r   r   r   FT)	rD   rE   rF   inttupler   r$   r;   rH   r4   r4   r2   r5   r   -  s(    =r   c                       s$   e Zd Z fddZdd Z  ZS )DistillCodecEncoderc                    s6   t    tddddd| _tdddddd	d
| _d S )Nr   )r   r   r   r   )r   r   r   r   )r   r   r<   r   )r?   r   r   r   i,     r   T)r?   r@   rS   r[   r+   r0   )r#   r$   r   encoderrY   
en_encoder)r*   r2   r4   r5   r$   p  s   
zDistillCodecEncoder.__init__c                 C   s   |  |}| |}|S rB   )r   r   rX   r4   r4   r5   r;     s   

zDistillCodecEncoder.forwardrM   r4   r4   r2   r5   r   o  s    r   )r^   r   r   local_attention.transformerr   r   r   distill_layersr   r	   r
   r   r   tconv.t_firstr   r   r   rI   rN   rY   rm   r   ResidualUnitr   ResidualLegacyUnitr   r   r   r4   r4   r4   r5   <module>   s$    N;,B