o
    
Û¾iÛ  ã                   @   sT   d dl mZ d dlmZmZmZmZ er d dlmZ d dl	m
Z
 G dd„ deƒZdS )é    )ÚPath)ÚTYPE_CHECKINGÚAnyÚProtocolÚoverload)ÚBatchEncoding)ÚChatCompletionMessageParamc                   @   s¨  e Zd ZeddddœdeeB dededB dedB dd f
d	d
„ƒZdefdd„Z	e
dee fdd„ƒZe
dee fdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd„ƒZe
defdd „ƒZdefd!d"„Zdefd#d$„Z		%		dFd&eee B d'edB d(ed)ed*edB dd+fd,d-„Zdeeef fd.d/„Zdeeef fd0d1„Z			%dGd&ed)edB d*edB d(edee f
d2d3„Z	dHd4ed5 d6eeeef  dB deee B fd7d8„Zed9edefd:d;„ƒZ ed9ee dee fd<d;„ƒZ d9eee B deee B fd=d;„Z d9ee defd>d?„Z!dId@ee eB dAedefdBdC„Z"	dId@ee dAedee fdDdE„Z#dS )JÚTokenizerLikeFN)Útrust_remote_codeÚrevisionÚdownload_dirÚpath_or_repo_idr
   r   r   Úreturnc                O   ó   t ‚©N©ÚNotImplementedError)Úclsr   r
   r   r   ÚargsÚkwargs© r   úL/home/ubuntu/.local/lib/python3.10/site-packages/vllm/tokenizers/protocol.pyÚfrom_pretrained   s   
zTokenizerLike.from_pretrainedc                 C   r   r   r   ©Úselfr   r   r   Únum_special_tokens_to_add   ó   z'TokenizerLike.num_special_tokens_to_addc                 C   r   r   r   r   r   r   r   Úall_special_tokens   ó   z TokenizerLike.all_special_tokensc                 C   r   r   r   r   r   r   r   Úall_special_ids    r   zTokenizerLike.all_special_idsc                 C   r   r   r   r   r   r   r   Úbos_token_id$   r   zTokenizerLike.bos_token_idc                 C   r   r   r   r   r   r   r   Úeos_token_id(   r   zTokenizerLike.eos_token_idc                 C   r   r   r   r   r   r   r   Úpad_token_id,   r   zTokenizerLike.pad_token_idc                 C   r   r   r   r   r   r   r   Úis_fast0   r   zTokenizerLike.is_fastc                 C   r   r   r   r   r   r   r   Ú
vocab_size4   r   zTokenizerLike.vocab_sizec                 C   r   r   r   r   r   r   r   Úmax_token_id8   r   zTokenizerLike.max_token_idc                 C   r   r   r   r   r   r   r   Úmax_chars_per_token<   r   z!TokenizerLike.max_chars_per_tokenc                 C   r   r   r   r   r   r   r   Útruncation_side@   r   zTokenizerLike.truncation_sidec                 C   s   t t| ƒƒS r   )ÚhashÚidr   r   r   r   Ú__hash__D   s   zTokenizerLike.__hash__c                 C   s   | j S r   )r$   r   r   r   r   Ú__len__G   s   zTokenizerLike.__len__TÚtextÚ	text_pairÚadd_special_tokensÚ
truncationÚ
max_lengthr   c                 C   r   r   r   )r   r,   r-   r.   r/   r0   r   r   r   Ú__call__J   s   zTokenizerLike.__call__c                 C   r   r   r   r   r   r   r   Ú	get_vocabT   r   zTokenizerLike.get_vocabc                 C   r   r   r   r   r   r   r   Úget_added_vocabW   r   zTokenizerLike.get_added_vocabc                 C   r   r   r   )r   r,   r/   r0   r.   r   r   r   ÚencodeZ   s   zTokenizerLike.encodeÚmessagesr   Útoolsc                 K   r   r   r   )r   r5   r6   r   r   r   r   Úapply_chat_templatec   s   z!TokenizerLike.apply_chat_templateÚtokensc                 C   ó   d S r   r   ©r   r8   r   r   r   Úconvert_tokens_to_idsk   r   z#TokenizerLike.convert_tokens_to_idsc                 C   r9   r   r   r:   r   r   r   r;   n   r   c                 C   r   r   r   r:   r   r   r   r;   q   r   c                 C   r   r   r   r:   r   r   r   Úconvert_tokens_to_stringt   r   z&TokenizerLike.convert_tokens_to_stringÚidsÚskip_special_tokensc                 C   r   r   r   ©r   r=   r>   r   r   r   Údecodew   r   zTokenizerLike.decodec                 C   r   r   r   r?   r   r   r   Úconvert_ids_to_tokensz   s   z#TokenizerLike.convert_ids_to_tokens)NTFN)NNTr   )F)$Ú__name__Ú
__module__Ú__qualname__ÚclassmethodÚstrr   Úboolr   Úintr   ÚpropertyÚlistr   r   r    r!   r"   r#   r$   r%   r&   r'   r*   r+   r1   Údictr2   r3   r4   r   r7   r   r;   r<   r@   rA   r   r   r   r   r	      s¾    úþüûúøú
þýüûú
ù
ûþýüû
úýþý

û" ýþýür	   N)Úpathlibr   Útypingr   r   r   r   Útransformersr   Úvllm.entrypoints.chat_utilsr   r	   r   r   r   r   Ú<module>   s   