o
    
۾i                     @   sz   d dl Z d dlmZ d dlmZ d dlmZ ddlmZ ddl	m
Z
 ddlmZmZ d	ed
efddZG dd deZdS )    N)Any)AutoTokenizer)ChatCompletionMessageParam   )TokenizerLike)encode_messages)HfTokenizerget_cached_tokenizer	tokenizerreturnc                    sV   t  }  t jG  fdddj}djj |_||_|S )zS
    Wraps a tokenizer to use the custom DeepSeek V3.2 chat template encoding.
    c                	       s   e Zd Z	dded deeeef  dB deee B fddZdefdd	Z	deffd
dZ
deeef f fddZfddZdS )z9get_deepseek_v32_tokenizer.<locals>._DeepseekV32TokenizerNmessagesr   toolsr   c                    s     dd}  dd}|p|}d}|sd}  d|}| }|d ur8t|dkr8|dddi ||d d	< |d
 d dk}t||d}	t|fi |	}
  ddrh fdddD }| j|
fddi|S |
S )NthinkingFenable_thinkingchatconversationr   rolesystemr   user)thinking_modedrop_thinkingtokenizeTc                    s   i | ]}| v r| | qS  r   ).0kkwargsr   P/home/ubuntu/.local/lib/python3.10/site-packages/vllm/tokenizers/deepseek_v32.py
<dictcomp>7   s    zaget_deepseek_v32_tokenizer.<locals>._DeepseekV32Tokenizer.apply_chat_template.<locals>.<dictcomp>)
truncation
max_lengthadd_special_tokens)getcopyleninsertdictr   encode)selfr   r   r   r   r   r   r   r   encode_config
prompt_strtokenizer_kwargsr   r   r   apply_chat_template   s8   
zMget_deepseek_v32_tokenizer.<locals>._DeepseekV32Tokenizer.apply_chat_templatec                 S   s   t | dS )N )r%   r(   r)   r   r   r   num_special_tokens_to_addB   s   zSget_deepseek_v32_tokenizer.<locals>._DeepseekV32Tokenizer.num_special_tokens_to_addc                    s     S Nr   r/   )added_vocab_sizetokenizer_vocab_sizer   r   __len__E   s   zAget_deepseek_v32_tokenizer.<locals>._DeepseekV32Tokenizer.__len__c                    s      S r1   )r$   r/   )added_vocabr   r   get_added_vocabI   s   zIget_deepseek_v32_tokenizer.<locals>._DeepseekV32Tokenizer.get_added_vocabc                    s
   t  ffS r1   )get_deepseek_v32_tokenizerr/   )r
   r   r   
__reduce__L   s   
zDget_deepseek_v32_tokenizer.<locals>._DeepseekV32Tokenizer.__reduce__r1   )__name__
__module____qualname__listr'   strr   intr-   r0   r4   r6   r8   r   r5   r2   r
   r3   r   r   _DeepseekV32Tokenizer   s    

(r@   DSV32)r$   r6   r%   
vocab_size	__class__r9   )r
   dsv32_tokenizerr@   r   r?   r   r7      s   
6r7   c                   @   s   e Zd ZedefddZdS )DeepseekV32Tokenizerr   c                 O   s   t j|i |}tt|S r1   )r   from_pretrainedr	   r7   )clsargsr   r
   r   r   r   rF   V   s   z$DeepseekV32Tokenizer.from_pretrainedN)r9   r:   r;   classmethodr   rF   r   r   r   r   rE   U   s    rE   )r$   typingr   transformersr   vllm.entrypoints.chat_utilsr   r.   r   deepseek_v32_encodingr   hfr   r	   r7   rE   r   r   r   r   <module>   s   F