o
    Si                     @   st   d Z ddlZddlZddlmZ ddlm  mZ ddej	de
defddZG d	d
 d
ejZG dd dejZdS )zA streamable transformer.    N'  	positionsdim
max_periodc                 C   sd   |d dksJ |d }t j|| jdddd}| |||d    }t jt |t |gddS )zKCreate time embedding for the given positions, target dimension `dim`.
       r   device   r   )torcharanger   viewcatcossin)r   r   r   half_dimadimphase r   O/home/ubuntu/.local/lib/python3.10/site-packages/encodec/modules/transformer.pycreate_sin_embedding   s   r   c                   @   s@   e Zd ZdejdejdefddZdejdejdefddZdS )	 StreamingTransformerEncoderLayerxx_pastpast_contextc                 C   sv   | j r| |}|| ||| }|| | | }||fS |}| || ||| }| || | }||fS )N)
norm_firstnorm1	_sa_block	_ff_blocknorm2)selfr   r   r   sa_inputr   r   r   forward   s   
z(StreamingTransformerEncoderLayer.forwardc                 C   s   |j \}}}|j \}}}|}tj||gdd}|}	tj||| |jddd}
tj|| |jddd}|
| }|dk||k@ }| j|||	| ddd }| |S )Nr	   r   r   r
   r   F)	attn_maskneed_weights)shaper   r   r   r   r   	self_attndropout1)r!   r   r   r   _THquerieskeysvaluesqueries_poskeys_posdeltavalid_accessr   r   r   r   ,   s    

z*StreamingTransformerEncoderLayer._sa_blockN)__name__
__module____qualname__r   Tensorintr#   r   r   r   r   r   r      s    r   c                       s   e Zd ZdZ			dd	ed
ededededededef fddZ		ddej	de
je
jej	  de
jeej	f fddZ  ZS )StreamingTransformerEncodera  TransformerEncoder with streaming support.

    Args:
        dim (int): dimension of the data.
        hidden_scale (int): intermediate dimension of FF module is this times the dimension.
        num_heads (int): number of heads.
        num_layers (int): number of layers.
        max_period (float): maxium period of cosines in the positional embedding.
        past_context (int or None): receptive field for the causal mask, infinite if None.
        gelu (bool): if true uses GeLUs, otherwise use ReLUs.
        norm_in (bool): normalize the input.
        dropout (float): dropout probability.
        **kwargs: See `nn.TransformerEncoderLayer`.
          @      r     T        hidden_scale	num_heads
num_layersr   r   gelunorm_indropoutc
              	      s   t    || dksJ t|| }|| _|| _|rtjntj}|  |r,t	|| _
nt | _
t | _t|D ]}| jt|||f|d|	d|
 q:d S )Nr   T)
activationbatch_firstrC   )super__init__r7   r   r   FrA   relunn	LayerNormrB   Identity
ModuleListlayersrangeappendr   )r!   r   r>   r?   r@   r   r   rA   rB   rC   kwargs
hidden_dimrD   idx	__class__r   r   rG   M   s,   


z$StreamingTransformerEncoder.__init__Nr   r   statesoffsetc                    s    j \}}}|d u r fddtdt| j D }tj| jdddd| }t||| j	d}g }	| 
   |  t|| jD ](\}
}| |
| j\ }tj|
|gdd}|	|d d | j d d d f  qB |	|| fS )Nc                    s&   g | ]}t  d d d df qS )Nr	   )r   
zeros_like).0r)   r   r   r   
<listcomp>j   s   & z7StreamingTransformerEncoder.forward.<locals>.<listcomp>r	   r   r
   )r   r   )r&   rO   lenrN   r   r   r   r   r   r   rB   zipr   r   rP   )r!   r   rV   rW   Br*   Cr   pos_emb	new_statelayer_statelayernew_layer_stater   rZ   r   r#   e   s    
&z#StreamingTransformerEncoder.forward)r9   r:   r;   r   r<   TTr=   )Nr   )r3   r4   r5   __doc__floatr7   boolrG   r   r6   tpOptionalListUnionr#   __classcell__r   r   rT   r   r8   >   s.    
r8   )r   )re   typingrh   r   torch.nnrJ   torch.nn.functional
functionalrH   r6   r7   rf   r   TransformerEncoderLayerr   Moduler8   r   r   r   r   <module>   s    