o
    -i                     @   sT   d dl mZ d dlmZmZmZmZ er d dlmZ d dl	m
Z
 G dd deZdS )    )Path)TYPE_CHECKINGAnyProtocoloverload)BatchEncoding)ChatCompletionMessageParamc                   @   s  e Zd ZedddddeeB dededB dedB dd f
d	d
ZdefddZ	e
dee fddZe
dee fddZe
defddZe
defddZe
defddZe
defddZe
defddZe
defddZe
defddZdefdd Zdefd!d"Z		#		dDd$eee B d%edB d&ed'ed(edB dd)fd*d+Zdeeef fd,d-Zdeeef fd.d/Z			#dEd$ed'edB d(edB d&edee f
d0d1Z	dFd2ed3 d4eeeef  dB deee B fd5d6Zed7edefd8d9Zed7ee dee fd:d9Zd7eee B deee B fd;d9Zd7ee defd<d=Z dGd>ee eB d?edefd@dAZ!	dGd>ee d?edee fdBdCZ"dS )HTokenizerLikeFN)trust_remote_coderevisiondownload_dirpath_or_repo_idr
   r   r   returnc                O      t NNotImplementedError)clsr   r
   r   r   argskwargs r   U/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/tokenizers/protocol.pyfrom_pretrained   s   
zTokenizerLike.from_pretrainedc                 C   r   r   r   selfr   r   r   num_special_tokens_to_add      z'TokenizerLike.num_special_tokens_to_addc                 C   r   r   r   r   r   r   r   all_special_tokens      z TokenizerLike.all_special_tokensc                 C   r   r   r   r   r   r   r   all_special_ids    r   zTokenizerLike.all_special_idsc                 C   r   r   r   r   r   r   r   bos_token_id$   r   zTokenizerLike.bos_token_idc                 C   r   r   r   r   r   r   r   eos_token_id(   r   zTokenizerLike.eos_token_idc                 C   r   r   r   r   r   r   r   pad_token_id,   r   zTokenizerLike.pad_token_idc                 C   r   r   r   r   r   r   r   is_fast0   r   zTokenizerLike.is_fastc                 C   r   r   r   r   r   r   r   
vocab_size4   r   zTokenizerLike.vocab_sizec                 C   r   r   r   r   r   r   r   max_token_id8   r   zTokenizerLike.max_token_idc                 C   r   r   r   r   r   r   r   truncation_side<   r   zTokenizerLike.truncation_sidec                 C   s   t t| S r   )hashidr   r   r   r   __hash__@   s   zTokenizerLike.__hash__c                 C   s   | j S r   )r$   r   r   r   r   __len__C   s   zTokenizerLike.__len__Ttext	text_pairadd_special_tokens
truncation
max_lengthr   c                 C   r   r   r   )r   r+   r,   r-   r.   r/   r   r   r   __call__F   s   zTokenizerLike.__call__c                 C   r   r   r   r   r   r   r   	get_vocabP   r   zTokenizerLike.get_vocabc                 C   r   r   r   r   r   r   r   get_added_vocabS   r   zTokenizerLike.get_added_vocabc                 C   r   r   r   )r   r+   r.   r/   r-   r   r   r   encodeV   s   zTokenizerLike.encodemessagesr   toolsc                 K   r   r   r   )r   r4   r5   r   r   r   r   apply_chat_template_   s   z!TokenizerLike.apply_chat_templatetokensc                 C      d S r   r   r   r7   r   r   r   convert_tokens_to_idsg   r   z#TokenizerLike.convert_tokens_to_idsc                 C   r8   r   r   r9   r   r   r   r:   j   r   c                 C   r   r   r   r9   r   r   r   r:   m   r   c                 C   r   r   r   r9   r   r   r   convert_tokens_to_stringp   r   z&TokenizerLike.convert_tokens_to_stringidsskip_special_tokensc                 C   r   r   r   r   r<   r=   r   r   r   decodes   r   zTokenizerLike.decodec                 C   r   r   r   r>   r   r   r   convert_ids_to_tokensv   s   z#TokenizerLike.convert_ids_to_tokens)NTFN)NNTr   )F)#__name__
__module____qualname__classmethodstrr   boolr   intr   propertylistr   r   r    r!   r"   r#   r$   r%   r&   r)   r*   r0   dictr1   r2   r3   r   r6   r   r:   r;   r?   r@   r   r   r   r   r	      s    





" r	   N)pathlibr   typingr   r   r   r   transformersr   vllm.entrypoints.chat_utilsr   r	   r   r   r   r   <module>   s   