o
    i~                     @   s   d dl Z d dlmZmZmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ edd	G d
d	 d	e jjZdS )    N)DictListOptionalTuple)tablesRWKV)	LayerNorm)make_source_mask)RWKVConvInputencoder_classesRWKVEncoderc                       s   e Zd ZdZ											dd	ed
ededee dee dedededededededdf fddZdefddZde	j
de	j
fddZdd Z  ZS ) r   a  RWKV encoder module.

    Based on https://arxiv.org/pdf/2305.13048.pdf.

    Args:
        vocab_size: Vocabulary size.
        output_size: Input/Output size.
        context_size: Context size for WKV computation.
        linear_size: FeedForward hidden size.
        attention_size: SelfAttention hidden size.
        normalization_type: Normalization layer type.
        normalization_args: Normalization layer arguments.
        num_blocks: Number of RWKV blocks.
        embed_dropout_rate: Dropout rate for embedding layer.
        att_dropout_rate: Dropout rate for the attention module.
        ffn_dropout_rate: Dropout rate for the feed-forward module.
          N                 
input_sizeoutput_sizecontext_sizelinear_sizeattention_size
num_blocksatt_dropout_rateffn_dropout_ratedropout_ratesubsampling_factortime_reduction_factorkernelreturnc              
      s   t    t|d d g|
|d| _|
| _du r!d ndu r)ntj fddtD | _	t
| _t
| _| _| _| _|| _dS )zConstruct a RWKVEncoder object.r      )conv_kernel_sizer   Nc                    s&   g | ]}t | d 	qS ))r   r   r   r   ).0block_idr   r   r   r   r   r   r   r    W/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/rwkv_bat/rwkv_encoder.py
<listcomp>E   s    z(RWKVEncoder.__init__.<locals>.<listcomp>)super__init__r   embedr   torchnn
ModuleListrangerwkv_blocksr	   
embed_norm
final_norm_output_sizer   r   r   )selfr   r   r   r   r   r   r   r   r   r   r   r   kwargs	__class__r%   r'   r*   $   s,   



zRWKVEncoder.__init__c                 C   s   | j S )Nr3   r4   r&   r&   r'   r   ^   s   zRWKVEncoder.output_sizexc                 C   s   |  \}}}|| j| j ksJ d|| j| j f t||j}| ||d\}}| |}|d	d}| j
rI| jD ]}||\}}q?n| |}| |}| jdkrr|dddd| jddf }t|d | jd }||dfS )zEncode source label sequences.

        Args:
            x: Encoder input sequences. (B, L)

        Returns:
            out: Encoder output sequences. (B, U, D)

        z:Context size is too short for current length: %d versus %dNr   r   )sizer   r   r
   todevicer+   r1   eqsumtrainingr0   
rwkv_inferr2   r   r,   floor_divide)r4   r:   x_len_lengthmaskolensblockr&   r&   r'   forwarda   s*   







zRWKVEncoder.forwardc                    s   j d  fddtdD  fddtdD }|d  d8  < g }tj d D ]$}d d |d d f }tjD ]\}}|||d	\}}qA|| q/tj|dd
}|S )Nr   c                    s   g | ]} j qS r&   r8   r#   ir9   r&   r'   r(      s    z*RWKVEncoder.rwkv_infer.<locals>.<listcomp>   c                    s.   g | ]}t j d | jft jjdqS )r   )dtyper=   )r,   zerosr   float32r=   rJ   
batch_sizehidden_sizesr4   xs_padr&   r'   r(      s    r   gKH9r   )state)dim)shaper/   	enumerater0   appendr,   cat)r4   rS   rT   xs_outtx_tidxrH   r&   rP   r'   rA      s   
	zRWKVEncoder.rwkv_infer)r   r   NNr   r   r   r   r   r   r   )__name__
__module____qualname____doc__intr   floatr*   r   r,   TensorrI   rA   __classcell__r&   r&   r6   r'   r      sV    	
:%)r,   typingr   r   r   r   funasr.registerr   funasr.models.rwkv_bat.rwkvr   $funasr.models.transformer.layer_normr	   *funasr.models.transformer.utils.nets_utilsr
   'funasr.models.rwkv_bat.rwkv_subsamplingr   registerr-   Moduler   r&   r&   r&   r'   <module>   s   
