o
    eiE                     @   s   d Z ddlZddlZddlZddlZddlmZ ddlmZ e	e
Zdddd	Zd
d Zdd Zdd ZG dd deZdgZdS )zTokenization classes for FSMT.    N   )PreTrainedTokenizer)loggingzvocab-src.jsonzvocab-tgt.jsonz
merges.txt)src_vocab_filetgt_vocab_filemerges_filec                 C   s6   t  }| d }| dd D ]}|||f |}q|S )z
    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length
    strings)
    r      N)setadd)wordpairs	prev_charchar r   h/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/fsmt/tokenization_fsmt.py	get_pairs"   s   r   c                 C   s  |  dd} tdd| } |  dd} |  dd} |  dd} |  d	d
} |  dd
} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  dd} |  d d!} |  d"d#} |  d$d%} |  d&d'} |  d(d)} |  d*d+} |  d,d-} td.d| } |  d/d0} |  d1d2} |  d3d4} |  d5d6} |  d7d8} |  d9d:} |  d;d<} |  d=d>} |  d?d@} | S )Azz
    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
    u   ，,u   。\s*z. u   、u   ”"u   “u   ∶:u   ：u   ？?u   《u   》u   ）)u   ！!u   （(u   ；;u   １1u   」u   「u   ０0u   ３3u   ２2u   ５5u   ６6u   ９9u   ７7u   ８8u   ４4u   ．\s*u   ～~u   ’'u   …z...u   ━-u   〈<u   〉>u   【[u   】]u   ％%)replaceresub)textr   r   r   replace_unicode_punct/   sJ   r0   c                 C   s8   g }| D ]}t |}|drq|| qd|S )zw
    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/remove-non-printing-char.perl
    C )unicodedatacategory
startswithappendjoin)r/   outputr   catr   r   r   remove_non_printing_charZ   s   


r:   c                
       sf  e Zd ZdZeZddgZ										d; fd
d	Zdee	e
f fddZede
fddZdd Zdd Zdd Zdd Zedd Zedd Zdd Zdd  Zd!d" Zd<d$d%Zd&d' Zd(d) Zd*d+ Z	d=d,ee
 d-ee
 dB dee
 fd.d/Z	d>d,ee
 d-ee
 dB d0edee
 f fd1d2Zd=d3e	d4e	dB dee	 fd5d6Z d7d8 Z!d9d: Z"  Z#S )?FSMTTokenizera	  
    Construct an FAIRSEQ Transformer tokenizer. Based on Byte-Pair Encoding. The tokenization process is the following:

    - Moses preprocessing and tokenization.
    - Normalizing all inputs text.
    - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols (like
      "__classify__") to a vocabulary.
    - The argument `langs` defines a pair of languages.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        langs (`List[str]`, *optional*):
            A list of two languages to translate from and to, for instance `["en", "ru"]`.
        src_vocab_file (`str`, *optional*):
            File containing the vocabulary for the source language.
        tgt_vocab_file (`st`, *optional*):
            File containing the vocabulary for the target language.
        merges_file (`str`, *optional*):
            File containing the merges.
        do_lower_case (`bool`, *optional*, defaults to `False`):
            Whether or not to lowercase the input when tokenizing.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.

    	input_idsattention_maskNF<unk><s></s><pad>c
                    s  zdd l }W n ty   tdw || _|| _|| _|| _|| _i | _i | _i | _	|r9t
|dkr9|\| _| _ntd| dt|dd}t|| _W d    n1 sXw   Y  t|dd}t|}dd	 | D | _W d    n1 s}w   Y  t|dd}| d
d d }W d    n1 sw   Y  dd |D }tt|tt
|| _i | _t jd|||||||||	d	|
 d S )Nr   nYou need to install sacremoses to use XLMTokenizer. See https://pypi.org/project/sacremoses/ for installation.   zFarg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got zw. Usually that means that tokenizer can't find a mapping for the given model path in  and other maps of this tokenizer.utf-8encodingc                 S      i | ]\}}||qS r   r   .0kvr   r   r   
<dictcomp>       z*FSMTTokenizer.__init__.<locals>.<dictcomp>
c                 S   s    g | ]}t | d d qS )NrC   )tuplesplit)rI   merger   r   r   
<listcomp>        z*FSMTTokenizer.__init__.<locals>.<listcomp>)	langsr   r   r   do_lower_case	unk_token	bos_token	sep_token	pad_tokenr   )
sacremosesImportErrorsmr   r   r   rV   cache_moses_punct_normalizercache_moses_tokenizercache_moses_detokenizerlensrc_langtgt_lang
ValueErroropenjsonloadencoderitemsdecoderreadrQ   dictziprange	bpe_rankscachesuper__init__)selfrU   r   r   r   rV   rW   rX   rY   rZ   kwargsr[   src_vocab_handletgt_vocab_handle	tgt_vocabmerges_handlemerges	__class__r   r   rr      s\   



zFSMTTokenizer.__init__returnc                 C   s   |   S N)get_src_vocabrs   r   r   r   	get_vocab   s   zFSMTTokenizer.get_vocabc                 C   s   | j S r}   )src_vocab_sizer   r   r   r   
vocab_size   s   zFSMTTokenizer.vocab_sizec                 C   2   || j vr| jj|d}|| j |< | j | |S Nlang)r^   r]   MosesPunctNormalizer	normalize)rs   r/   r   punct_normalizerr   r   r   moses_punct_norm      

zFSMTTokenizer.moses_punct_normc                 C   s:   || j vr| jj|d}|| j |< | j | j|ddddS )Nr   TF)aggressive_dash_splits
return_strescape)r_   r]   MosesTokenizertokenize)rs   r/   r   moses_tokenizerr   r   r   moses_tokenize   s   


zFSMTTokenizer.moses_tokenizec                 C   r   r   )r`   r]   MosesDetokenizer
detokenize)rs   tokensr   moses_detokenizerr   r   r   moses_detokenize   r   zFSMTTokenizer.moses_detokenizec                 C   s    t |}| ||}t|}|S r}   )r0   r   r:   )rs   r/   r   r   r   r   moses_pipeline   s   zFSMTTokenizer.moses_pipelinec                 C   
   t | jS r}   )ra   rh   r   r   r   r   r        
zFSMTTokenizer.src_vocab_sizec                 C   r   r}   )ra   rj   r   r   r   r   tgt_vocab_size
  r   zFSMTTokenizer.tgt_vocab_sizec                 C      t | jfi | jS r}   )rl   rh   added_tokens_encoderr   r   r   r   r~        zFSMTTokenizer.get_src_vocabc                 C   r   r}   )rl   rj   added_tokens_decoderr   r   r   r   get_tgt_vocab  r   zFSMTTokenizer.get_tgt_vocabc           
         s~  t |d d |d d f }| jv r j| S t|}|s#|d S 	 t| fddd}| jvr4ny|\}}g }d}|t|k rz|||}	W n ty\   |||d   Y n?w ||||	  |	}|| |kr|t|d k r||d  |kr|	||  |d	7 }n|	||  |d7 }|t|k sBt |}|}t|dkrnt|}q$d

|}|dkrd}| j|< |S )NrO   </w>Tc                    s    j | tdS )Ninf)ro   getfloat)pairr   r   r   <lambda>  s    z#FSMTTokenizer.bpe.<locals>.<lambda>keyr   r   rC    z
  </w>z
</w>)rP   rp   r   minro   ra   indexrd   extendr6   r7   )
rs   tokenr   r   bigramfirstsecondnew_wordijr   r   r   bpe  sN   


,


zFSMTTokenizer.bpeenc                 C   sn   | j }| jr
| }|r| }n| j||d}| j||d}g }|D ]}|r4|t| |d q#|S )av  
        Tokenize a string given language code using Moses.

        Details of tokenization:

            - [sacremoses](https://github.com/alvations/sacremoses): port of Moses
            - Install with `pip install sacremoses`

        Args:
            - lang: ISO language code (default = 'en') (string). Languages should belong of the model supported
              languages. However, we don't enforce it.
            - bypass_tokenizer: Allow users to preprocess and tokenize the sentences externally (default = False)
              (bool). If True, we only apply BPE.

        Returns:
            List of tokens.
        r   r   )	rb   rV   lowerrQ   r   r   r   listr   )rs   r/   r   bypass_tokenizersplit_tokensr   r   r   r   	_tokenize@  s   
zFSMTTokenizer._tokenizec                 C   s   | j || j | jS )z0Converts a token (str) in an id using the vocab.)rh   r   rW   )rs   r   r   r   r   _convert_token_to_idg  s   z"FSMTTokenizer._convert_token_to_idc                 C   s   | j || jS )z=Converts an index (integer) in a token (str) using the vocab.)rj   r   rW   )rs   r   r   r   r   _convert_id_to_tokenk  s   z"FSMTTokenizer._convert_id_to_tokenc                 C   s.   dd |D }d | }| || j}|S )z:Converts a sequence of tokens (string) in a single string.c                 S   s    g | ]}| d d dd qS )r   r2   r   )r,   )rI   tr   r   r   rS   s  rT   z:FSMTTokenizer.convert_tokens_to_string.<locals>.<listcomp>r2   )r7   rQ   r   rc   )rs   r   r/   r   r   r   convert_tokens_to_stringo  s   z&FSMTTokenizer.convert_tokens_to_stringtoken_ids_0token_ids_1c                 C   s(   | j g}|du r|| S || | | S )a  
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A FAIRSEQ Transformer sequence has the following format:

        - single sequence: `<s> X </s>`
        - pair of sequences: `<s> A </s> B </s>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        N)sep_token_id)rs   r   r   sepr   r   r    build_inputs_with_special_tokensy  s   z.FSMTTokenizer.build_inputs_with_special_tokensalready_has_special_tokensc                    sZ   |rt  j||ddS |dur#dgt| dg dgt|  dg S dgt| dg S )a  
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        T)r   r   r   Nr   r   )rq   get_special_tokens_maskra   )rs   r   r   r   rz   r   r   r     s   (z%FSMTTokenizer.get_special_tokens_masksave_directoryfilename_prefixc              	   C   s  t j|std| d d S t j||r|d ndtd  }t j||r,|d ndtd  }t j||r=|d ndtd  }t|dd	d
}|t	j
| jddddd  W d    n1 sew   Y  t|dd	d
 }dd | j D }|t	j
|ddddd  W d    n1 sw   Y  d}t|dd	d
5}	t| j dd dD ]!\}
}||krtd| d |}|	d|
d  |d7 }qW d    n1 sw   Y  |||fS )NzVocabulary path (z) should be a directoryr&   r2   r   r   r   wrD   rE   rC   TF)indent	sort_keysensure_asciirN   c                 S   rG   r   r   rH   r   r   r   rL     rM   z1FSMTTokenizer.save_vocabulary.<locals>.<dictcomp>r   c                 S   s   | d S )Nr   r   )kvr   r   r   r     s    z/FSMTTokenizer.save_vocabulary.<locals>.<lambda>r   zSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!r   r   )ospathisdirloggererrorr7   VOCAB_FILES_NAMESre   writerf   dumpsrh   rj   ri   sortedro   warning)rs   r   r   r   r   r   frw   r   writer
bpe_tokenstoken_indexr   r   r   save_vocabulary  s@    


zFSMTTokenizer.save_vocabularyc                 C   s   | j  }d |d< |S )Nr]   )__dict__copy)rs   stater   r   r   __getstate__  s   
zFSMTTokenizer.__getstate__c                 C   s4   || _ zdd l}W n ty   tdw || _d S )Nr   rB   )r   r[   r\   r]   )rs   dr[   r   r   r   __setstate__  s   
zFSMTTokenizer.__setstate__)	NNNNFr>   r?   r@   rA   )r   Fr}   )NF)$__name__
__module____qualname____doc__r   vocab_files_namesmodel_input_namesrr   rl   strintr   propertyr   r   r   r   r   r   r   r~   r   r   r   r   r   r   r   r   boolr   rP   r   r   r   __classcell__r   r   rz   r   r;   p   sj    -C


,'


 $r;   )r   rf   r   r-   r3   tokenization_pythonr   utilsr   
get_loggerr   r   r   r   r0   r:   r;   __all__r   r   r   r   <module>   s&   
+  
x