o
    پi                     @   sP   d dl Z d dlmZ e eZdd Zdd Zdd ZG d	d
 d
Z	dd Z
dS )    N)envsc                 C   s8   t j s| S t| rtdt|   t| S | S )Nz8Applying special tokens cache patch for Kimi tokenizer: )	r   SGLANG_PATCH_TOKENIZERget_is_kimi_tiktoken_tokenizerloggerinfotype_SpecialTokensCachePatcherpatch	tokenizer r   T/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/utils/patch_tokenizer.pypatch_tokenizer   s   

r   c                 C   s
   t | S N)r	   unpatchr   r   r   r   unpatch_tokenizer   s   
r   c                 C   s(   t | }|j}|jpd}|dkod|v S )N TikTokenTokenizertokenization_kimi)r   __name__
__module__)r   cls
class_namemodule_namer   r   r   r      s   
r   c                   @   s0   e Zd ZdZdZdZedd Zedd ZdS )	r	   _sglang_special_tokens_patched_sglang_cached_special_tokens_sglang_cached_special_idsc                    s   t | t | jdr|S  jj _ jj _ j _	 j
 _t| j j}t| j j}dd }d fdd	}| _| _| _| _
t | jd |S )NFc                 _   s
   	J d)NFzGCannot modify special tokens after patch. Call unpatch_tokenizer first.r   )selfargskwargsr   r   r   patched_add_special_tokens:   s   zD_SpecialTokensCachePatcher.patch.<locals>.patched_add_special_tokensc                    s   |rJ d j | |ddS )NzDCannot add special tokens after patch. Call unpatch_tokenizer first.F)special_tokens)_original_add_tokens)r   
new_tokensr"   tokenizer_clsr   r   patched_add_tokens?   s   z<_SpecialTokensCachePatcher.patch.<locals>.patched_add_tokensT)F)r   getattr_PATCHED_FLAGall_special_tokensfget_original_all_special_tokensall_special_ids_original_all_special_idsadd_special_tokens_original_add_special_tokens
add_tokensr#   _make_cached_property_CACHED_TOKENS_ATTR_CACHED_IDS_ATTRsetattr)r   r   patched_all_special_tokenspatched_all_special_idsr!   r'   r   r%   r   r
   %   s,   
z _SpecialTokensCachePatcher.patchc                 C   s   t |}t|| jds|S t|j|_t|j|_|j|_	|j
|_|`|`|`|`
t|| j | j| jfD ]}t||rAt|| q5td|j  |S )NFz#Unpatched special tokens cache for )r   r(   r)   propertyr,   r*   r.   r-   r0   r/   r#   r1   delattrr3   r4   hasattrr   r   r   )r   r   r&   attrr   r   r   r   O   s,   

z"_SpecialTokensCachePatcher.unpatchN)	r   r   __qualname__r)   r3   r4   classmethodr
   r   r   r   r   r   r	       s    
)r	   c                    s   t  fdd}|S )Nc                    s*   t |  d d u rt|  |  t |  S r   )r(   r5   )r   
cache_attroriginal_fnr   r   cached_propn   s   
z*_make_cached_property.<locals>.cached_prop)r8   )r?   r@   rA   r   r>   r   r2   m   s   r2   )loggingsglang.srt.environr   	getLoggerr   r   r   r   r   r	   r2   r   r   r   r   <module>   s    
M