o
    ei
                     @   sF   d Z ddlZddlmZ ddlmZ eeZG dd dZ	dgZ
dS )	zTokenization classes for RAG.    N   )logging   )	RagConfigc                   @   sP   e Zd Zdd Zdd Zedd Zdd Zd	d
 Zdd Z	dd Z
dd ZdS )RagTokenizerc                 C   s   || _ || _| j | _d S N)question_encoder	generatorcurrent_tokenizer)selfr   r	    r   f/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/rag/tokenization_rag.py__init__   s   zRagTokenizer.__init__c                 C   sb   t j|rtd| dt j|dd t j|d}t j|d}| j| | j| d S )NzProvided path (z#) should be a directory, not a fileT)exist_okquestion_encoder_tokenizergenerator_tokenizer)	ospathisfile
ValueErrormakedirsjoinr   save_pretrainedr	   )r   save_directoryquestion_encoder_pathgenerator_pathr   r   r   r      s   zRagTokenizer.save_pretrainedc                 K   sZ   ddl m} |dd }|d u rt|}|j||jdd}|j||jdd}| ||dS )N   )AutoTokenizerconfigr   )r   	subfolderr   )r   r	   )auto.tokenization_autor   popr   from_pretrainedr   r	   )clspretrained_model_name_or_pathkwargsr   r   r   r	   r   r   r   r"   (   s   
zRagTokenizer.from_pretrainedc                 O   s   | j |i |S r   )r
   r   argsr%   r   r   r   __call__:   s   zRagTokenizer.__call__c                 O      | j j|i |S r   )r	   batch_decoder&   r   r   r   r*   =      zRagTokenizer.batch_decodec                 O   r)   r   )r	   decoder&   r   r   r   r,   @   r+   zRagTokenizer.decodec                 C      | j | _d S r   )r   r
   r   r   r   r   _switch_to_input_modeC      z"RagTokenizer._switch_to_input_modec                 C   r-   r   )r	   r
   r.   r   r   r   _switch_to_target_modeF   r0   z#RagTokenizer._switch_to_target_modeN)__name__
__module____qualname__r   r   classmethodr"   r(   r*   r,   r/   r1   r   r   r   r   r      s    	
r   )__doc__r   utilsr   configuration_ragr   
get_loggerr2   loggerr   __all__r   r   r   r   <module>   s   

1