o
    i                     @   sT   d dl mZ d dlmZmZmZ d dlZd dlm	Z	 d dl
mZ G dd deZdS )    )Path)IterableListUnionN)check_argument_types)AbsTokenizerc                   @   s^   e Zd Zdeeef fddZdd Zdd Zded	e	e fd
dZ
dee d	efddZdS )SentencepiecesTokenizermodelc                 C   s   t  sJ t|| _d | _d S N)r   strr	   sp)selfr	    r   X/home/ubuntu/.local/lib/python3.10/site-packages/espnet2/text/sentencepiece_tokenizer.py__init__   s   


z SentencepiecesTokenizer.__init__c                 C   s   | j j d| j dS )Nz(model="z"))	__class____name__r	   r   r   r   r   __repr__   s   z SentencepiecesTokenizer.__repr__c                 C   s*   | j d u rt | _ | j | j d S d S r
   )r   spmSentencePieceProcessorloadr	   r   r   r   r   _build_sentence_piece_processor   s   

z7SentencepiecesTokenizer._build_sentence_piece_processorlinereturnc                 C   s   |    | j|S r
   )r   r   EncodeAsPieces)r   r   r   r   r   text2tokens   s   z#SentencepiecesTokenizer.text2tokenstokensc                 C   s   |    | jt|S r
   )r   r   DecodePieceslist)r   r   r   r   r   tokens2text"   s   z#SentencepiecesTokenizer.tokens2textN)r   
__module____qualname__r   r   r   r   r   r   r   r   r   r    r   r   r   r   r   
   s    
r   )pathlibr   typingr   r   r   sentencepiecer   	typeguardr   espnet2.text.abs_tokenizerr   r   r   r   r   r   <module>   s    