o
    i                     @   sl   d dl mZmZmZ d dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ G dd deZdS )	    )AnyListTupleN)AbsLM)PositionalEncoding)Encoder)subsequent_maskc                       s   e Zd Z							d#ded	ed
edededededef fddZdd Zdej	ddde
ej	df fddZdej	dedej	de
ej	ef fddZdej	dee d ej	de
ej	ee f fd!d"Z  ZS )$TransformerLMN                     ?
vocab_sizepos_enc
embed_unitatt_unitheadunitlayerdropout_ratec	           
   
      sr   t    |dkrt}	n|d u rdd }	ntd| t||| _t||||||d|	d| _t	||| _
d S )N
sinusoidalc                  _   s   t  S )N)nn
Sequential)argskwargs r   M/home/ubuntu/.local/lib/python3.10/site-packages/espnet2/lm/transformer_lm.pypos_enc_class   s   z-TransformerLM.__init__.<locals>.pos_enc_classzunknown pos-enc option: linear)idimattention_dimattention_headslinear_units
num_blocksr   input_layerr   )super__init__r   
ValueErrorr   	Embeddingembedr   encoderLineardecoder)
selfr   r   r   r   r   r   r   r   r   	__class__r   r   r(      s$   


zTransformerLM.__init__c                 C   s0   |dk}t |d|jdd}|d|@ S )Nr   )device)r   sizer3   	unsqueeze)r/   	ys_in_padys_maskmr   r   r   _target_mask0   s   zTransformerLM._target_maskinputhiddenreturnc                 C   s6   |  |}| |}| ||\}}| |}|dfS )zCompute LM loss value from buffer sequences.

        Args:
            input (torch.Tensor): Input ids. (batch, len)
            hidden (torch.Tensor): Target ids. (batch, len)

        N)r+   r:   r,   r.   )r/   r;   r<   xmaskh_yr   r   r   forward5   s
   


zTransformerLM.forwardrB   stater>   c                 C   s^   | d}| jj| || ||d\}}}| |dddf }|jddd}||fS )a  Score new token.

        Args:
            y (torch.Tensor): 1D torch.int64 prefix tokens.
            state: Scorer state for prefix tokens
            x (torch.Tensor): encoder feature that generates ys.

        Returns:
            tuple[torch.Tensor, Any]: Tuple of
                torch.float32 scores for next token (vocab_size)
                and next state for ys

        r   cacheNr2   dim)r6   r,   forward_one_stepr+   r:   r.   log_softmaxsqueeze)r/   rB   rD   r>   r@   rA   rF   logpr   r   r   scoreC   s   
zTransformerLM.scoreysstatesxsc           	         s   t | t | jjd du rd}n fddtD }| jj| || ||d\}}| |dddf }|jdd}fddt D }||fS )	a  Score new token batch.

        Args:
            ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen).
            states (List[Any]): Scorer states for prefix tokens.
            xs (torch.Tensor):
                The encoder feature that generates ys (n_batch, xlen, n_feat).

        Returns:
            tuple[torch.Tensor, List[Any]]: Tuple of
                batchfied scores for next token with shape of `(n_batch, vocab_size)`
                and next state list for ys.

        r   Nc                    s*   g | ] t  fd dtD qS )c                       g | ]}|   qS r   r   ).0b)irO   r   r   
<listcomp>t       8TransformerLM.batch_score.<locals>.<listcomp>.<listcomp>)torchstackrangerR   )n_batchrO   )rT   r   rU   s   s    z-TransformerLM.batch_score.<locals>.<listcomp>rE   r2   rG   c                    s$   g | ]  fd dt D qS )c                    rQ   r   r   )rR   rT   )rS   rO   r   r   rU      rV   rW   )rZ   r[   )n_layersrO   )rS   r   rU      s   $ )	lenr,   encodersrZ   rI   r+   r:   r.   rJ   )	r/   rN   rO   rP   batch_stater@   rA   rL   
state_listr   )r\   r]   rO   r   batch_score[   s   zTransformerLM.batch_score)Nr
   r   r   r   r   r   )__name__
__module____qualname__intstrfloatr(   r:   rX   Tensorr   rC   r   rM   r   rb   __classcell__r   r   r0   r   r	      sZ    	#"
r	   )typingr   r   r   rX   torch.nnr   espnet2.lm.abs_modelr   1espnet.nets.pytorch_backend.transformer.embeddingr   /espnet.nets.pytorch_backend.transformer.encoderr   ,espnet.nets.pytorch_backend.transformer.maskr   r	   r   r   r   r   <module>   s    