o
    -i                     @   s  d dl Zd dlmZmZ d dlmZ d dlmZ d dl	m
Z
 d dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZ d d
lmZ ddlmZ e
r`d dlm Z m!Z! ee"Z#dddddZ$eG dd dZ%e%dd e$& D Z'dddde(eB ddde(fddZ)ee)Z*d2d"d#Z+ed$eed%Z,ed&ddd'de(eB d(e-e, d)e.d*e(dB d+e(dB d,e,fd-d.Z/ee/Z0d3d0d1Z1dS )4    N)	dataclassfield)	lru_cache)Path)TYPE_CHECKING)TypeVarassert_never)init_logger)check_gguf_fileget_gguf_file_path_from_hfis_ggufis_remote_ggufsplit_remote_gguf)list_filtered_repo_files)resolve_obj_by_qualname   )TokenizerLike)ModelConfig
RunnerType)deepseek_v32DeepseekV32Tokenizer)grok2Grok2Tokenizer)hfCachedHfTokenizer)mistralMistralTokenizer)r   r   r   r   c                   @   sr   e Zd ZU eedZeeeeef f ed< dedededdfdd	Z	dede
e fd
dZdedefddZdS )_TokenizerRegistry)default_factory
tokenizerstokenizer_modemodule
class_namereturnNc                 C   s,   || j v rtd||| ||f| j |< d S )NzT%s.%s is already registered for tokenizer_mode=%r. It is overwritten by the new one.)r   loggerwarningselfr    r!   r"    r(   U/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/tokenizers/registry.pyregister-   s   
z_TokenizerRegistry.registerc                 C   sP   || j vrtd|d| j | \}}td| d| t| d| S )Nz+No tokenizer registered for tokenizer_mode=.zLoading z for tokenizer_mode=)r   
ValueErrorr$   
debug_oncer   r&   r(   r(   r)   load_tokenizer_cls;   s
   
z%_TokenizerRegistry.load_tokenizer_clsc                 O   s   |  |}|j|i |S )N)r.   from_pretrained)r'   r    argskwargstokenizer_clsr(   r(   r)   load_tokenizerD   s   
z!_TokenizerRegistry.load_tokenizer)__name__
__module____qualname__r   dictr   strtuple__annotations__r*   typer   r.   r3   r(   r(   r(   r)   r   (   s
   
 "	r   c                 C   s$   i | ]\}\}}|d | |fqS )zvllm.tokenizers.r(   ).0modemod_relnamecls_namer(   r(   r)   
<dictcomp>J   s    
r@   generateauto)runner_typer    tokenizer_namerC   r   r    c                O   s  | d}| d}tjrEddlm} ddlm} t|  sE|| | |t	| ||t
jjg dd}	|	} W d    n1 s@w   Y  t| rot| rZt| j|d< t| j} nt| rot| \} }
t| |
|d	}||d< d
|vr|dks{|dkrd|d
< n|dkrd|d
< nt| |dkr| ddrtdd}d|d< |dkrtjdrddg}tt	| ||d}t|dkrd}|dkrdg}tt	| ||d}t|dkrd}|dkrd}|| ||fS )Nrevisiondownload_dirr   )snapshot_download)get_lock)z.*.ptz.*.safetensorsz.*.bin)model_id	cache_dirrE   local_files_onlyignore_file_pattern	gguf_file)rE   truncation_siderA   draftleftpoolingrightslowuse_fastFz5Cannot use the fast tokenizer in slow tokenizer mode.r   rB   mistral_commonztekken.jsonztokenizer.model.v*)model_name_or_pathallow_patternsrE   r   ztokenizer.tok.jsonr   )getenvsVLLM_USE_MODELSCOPE modelscope.hub.snapshot_downloadrG   -vllm.model_executor.model_loader.weight_utilsrH   r   existsr8   huggingface_hub	constantsHF_HUB_OFFLINEr   r
   nameparentr   r   r   r   r,   	importlibutil	find_specr   len)rD   rC   r    r0   r1   rE   rF   rG   rH   tokenizer_path
quant_typerM   rW   
files_listr(   r(   r)   resolve_tokenizer_argsQ   sx   



rj   configr   c                 K   s$   t | jf| j| j| j| jd|S N)rC   r    rE   trust_remote_code)cached_resolve_tokenizer_args	tokenizerrC   r    tokenizer_revisionrm   )rk   r1   r(   r(   r)   tokenizer_args_from_config   s   rq   _T)bounddefaultF)r2   rm   rE   rF   r2   rm   rE   rF   r#   c          
      O   sj   t | g|R |||d|\}} }}|tkrt|}n|}|j| g|R i |}	|	js3td |	S )zHGets a tokenizer for the given model name via HuggingFace or ModelScope.)rm   rE   rF   ziUsing a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead.)rn   r   TokenizerRegistryr.   r/   is_fastr$   r%   )
rD   r2   rm   rE   rF   r0   r1   r    tokenizer_cls_ro   r(   r(   r)   get_tokenizer   s(   
	rx   model_configc                 K   s.   | j rd S t| jf| j| j| j| jd|S rl   )skip_tokenizer_initcached_get_tokenizerro   rC   r    rp   rm   )ry   r1   r(   r(   r)   cached_tokenizer_from_config   s   r|   )rk   r   )ry   r   )2importlib.utilrc   dataclassesr   r   	functoolsr   pathlibr   typingr   r^   typing_extensionsr   r   	vllm.envsrY   vllm.loggerr	   "vllm.transformers_utils.gguf_utilsr
   r   r   r   r   "vllm.transformers_utils.repo_utilsr   vllm.utils.import_utilsr   protocolr   vllm.config.modelr   r   r4   r$   _VLLM_TOKENIZERSr   itemsru   r8   rj   rn   rq   rr   r;   boolrx   r{   r|   r(   r(   r(   r)   <module>   sv    
\

"