o
    i                     @   s|   d dl mZ d dl mZ d dl mZ d dlZd dlmZ d dlmZ d dl	m
Z d dlmZ d dlmZ G d	d
 d
eZdS )    )Any)List)TupleN)PositionalEncoding)TransformerEncoder_s0)subsequent_mask)AbsLMc                       s   e Zd Z							d#ded	ed
edededededef fddZdd Zdej	ddde
ej	df fddZdej	dedej	de
ej	ef fddZdej	dee d ej	de
ej	ee f fd!d"Z  ZS )$TransformerLMN                     ?
vocab_sizepos_enc
embed_unitatt_unitheadunitlayerdropout_ratec	           
   
      sr   t    |dkrt}	n|d u rdd }	ntd| t||| _t||||||d|	d| _t	||| _
d S )N
sinusoidalc                  _   s   t  S )N)nn
Sequential)argskwargs r   _/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/language_model/transformer_lm.pypos_enc_class   s   z-TransformerLM.__init__.<locals>.pos_enc_classzunknown pos-enc option: linear)idimattention_dimattention_headslinear_units
num_blocksr   input_layerr   )super__init__r   
ValueErrorr   	EmbeddingembedEncoderencoderLineardecoder)
selfr   r   r   r   r   r   r   r   r   	__class__r   r   r(      s$   


zTransformerLM.__init__c                 C   s0   |dk}t |d|jdd}|d|@ S )Nr   )device)r   sizer4   	unsqueeze)r0   	ys_in_padys_maskmr   r   r   _target_mask2   s   zTransformerLM._target_maskinputhiddenreturnc                 C   s6   |  |}| |}| ||\}}| |}|dfS )zCompute LM loss value from buffer sequences.

        Args:
            input (torch.Tensor): Input ids. (batch, len)
            hidden (torch.Tensor): Target ids. (batch, len)

        N)r+   r;   r-   r/   )r0   r<   r=   xmaskh_yr   r   r   forward7   s
   


zTransformerLM.forwardrC   stater?   c                 C   s^   | d}| jj| || ||d\}}}| |dddf }|jddd}||fS )a  Score new token.

        Args:
            y (torch.Tensor): 1D torch.int64 prefix tokens.
            state: Scorer state for prefix tokens
            x (torch.Tensor): encoder feature that generates ys.

        Returns:
            tuple[torch.Tensor, Any]: Tuple of
                torch.float32 scores for next token (vocab_size)
                and next state for ys

        r   cacheNr3   dim)r7   r-   forward_one_stepr+   r;   r/   log_softmaxsqueeze)r0   rC   rE   r?   rA   rB   rG   logpr   r   r   scoreE   s   
zTransformerLM.scoreysstatesxsc           	         s   t | t | jjd du rd}n fddtD }| jj| || ||d\}}| |dddf }|jdd}fddt D }||fS )	a  Score new token batch.

        Args:
            ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen).
            states (List[Any]): Scorer states for prefix tokens.
            xs (torch.Tensor):
                The encoder feature that generates ys (n_batch, xlen, n_feat).

        Returns:
            tuple[torch.Tensor, List[Any]]: Tuple of
                batchfied scores for next token with shape of `(n_batch, vocab_size)`
                and next state list for ys.

        r   Nc                    s*   g | ] t  fd dtD qS )c                       g | ]}|   qS r   r   ).0b)irP   r   r   
<listcomp>t       8TransformerLM.batch_score.<locals>.<listcomp>.<listcomp>)torchstackrangerS   )n_batchrP   )rU   r   rV   s   s    z-TransformerLM.batch_score.<locals>.<listcomp>rF   r3   rH   c                    s$   g | ]  fd dt D qS )c                    rR   r   r   )rS   rU   )rT   rP   r   r   rV      rW   rX   )r[   r\   )n_layersrP   )rT   r   rV      s   $ )	lenr-   encodersr[   rJ   r+   r;   r/   rK   )	r0   rO   rP   rQ   batch_staterA   rB   rM   
state_listr   )r]   r^   rP   r   batch_score[   s   zTransformerLM.batch_score)Nr
   r   r   r   r   r   )__name__
__module____qualname__intstrfloatr(   r;   rY   Tensorr   rD   r   rN   r   rc   __classcell__r   r   r1   r   r	      sJ    	#"(r	   )typingr   r   r   rY   torch.nnr   #funasr.models.transformer.embeddingr   )funasr.models.encoder.transformer_encoderr   r,   $funasr.models.transformer.utils.maskr   funasr.train.abs_modelr   r	   r   r   r   r   <module>   s    