o
    
۾i7                     @   s  d dl mZmZ d dlmZ d dlmZ d dlmZ d dl	Z	d dl
mZmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZmZ d d
lmZ ddlmZ er^d dlmZm Z  ee!Z"dddddZ#eG dd dZ$e$dd e#% D Z&dddde'eB ddde'fddZ(ee(Z)d2d"d#Z*ed$eed%Z+ed&ddd'de'eB d(e,e+ d)e-d*e'dB d+e'dB d,e+fd-d.Z.ee.Z/d3d0d1Z0dS )4    )	dataclassfield)	lru_cache)Path)TYPE_CHECKINGN)TypeVarassert_never)init_logger)check_gguf_fileget_gguf_file_path_from_hfis_ggufis_remote_ggufsplit_remote_gguf)any_pattern_in_repo_filesis_mistral_model_repo)resolve_obj_by_qualname   )TokenizerLike)ModelConfig
RunnerType)deepseek_v32DeepseekV32Tokenizer)grok2Grok2Tokenizer)hfCachedHfTokenizer)mistralMistralTokenizer)r   r   r   r   c                   @   sr   e Zd ZU eedZeeeeef f ed< dedededdfdd	Z	dede
e fd
dZdedefddZdS )_TokenizerRegistry)default_factory
tokenizerstokenizer_modemodule
class_namereturnNc                 C   s,   || j v rtd||| ||f| j |< d S )NzT%s.%s is already registered for tokenizer_mode=%r. It is overwritten by the new one.)r    loggerwarningselfr!   r"   r#    r)   L/home/ubuntu/.local/lib/python3.10/site-packages/vllm/tokenizers/registry.pyregister/   s   
z_TokenizerRegistry.registerc                 C   sP   || j vrtd|d| j | \}}td| d| t| d| S )Nz+No tokenizer registered for tokenizer_mode=.zLoading z for tokenizer_mode=)r    
ValueErrorr%   
debug_oncer   r'   r)   r)   r*   load_tokenizer_cls=   s
   
z%_TokenizerRegistry.load_tokenizer_clsc                 O   s   |  |}|j|i |S )N)r/   from_pretrained)r(   r!   argskwargstokenizer_clsr)   r)   r*   load_tokenizerF   s   
z!_TokenizerRegistry.load_tokenizer)__name__
__module____qualname__r   dictr    strtuple__annotations__r+   typer   r/   r4   r)   r)   r)   r*   r   *   s
   
 "	r   c                 C   s$   i | ]\}\}}|d | |fqS )zvllm.tokenizers.r)   ).0modemod_relnamecls_namer)   r)   r*   
<dictcomp>L   s    
rA   generateauto)runner_typer!   tokenizer_namerD   r   r!   c                O   s  | d}| d}tjrEddlm} ddlm} t|  sE|| | |t	| ||t
jjg dd}	|	} W d    n1 s@w   Y  t| rot| rZt| j|d< t| j} nt| rot| \} }
t| |
|d	}||d< d
|vr|dks{|dkrd|d
< n|dkrd|d
< nt| |dkr| ddrtdd}d|d< |dkrtt	| |drtt	| ddg|drd}|dkrtt	| dg|drd}|dkrd}|| ||fS )Nrevisiondownload_dirr   )snapshot_download)get_lock)z.*.ptz.*.safetensorsz.*.bin)model_id	cache_dirrF   local_files_onlyignore_file_pattern	gguf_file)rF   truncation_siderB   draftleftpoolingrightslowuse_fastFz5Cannot use the fast tokenizer in slow tokenizer mode.r   rC   )model_name_or_pathrF   ztekken.jsonztokenizer.model.v*)rV   allow_patternsrF   r   ztokenizer.tok.jsonr   )getenvsVLLM_USE_MODELSCOPE modelscope.hub.snapshot_downloadrH   -vllm.model_executor.model_loader.weight_utilsrI   r   existsr9   huggingface_hub	constantsHF_HUB_OFFLINEr   r
   nameparentr   r   r   r   r-   r   r   )rE   rD   r!   r1   r2   rF   rG   rH   rI   tokenizer_path
quant_typerN   r)   r)   r*   resolve_tokenizer_argsS   sx   





re   configr   c                 K   s$   t | jf| j| j| j| jd|S N)rD   r!   rF   trust_remote_code)cached_resolve_tokenizer_args	tokenizerrD   r!   tokenizer_revisionrh   )rf   r2   r)   r)   r*   tokenizer_args_from_config   s   rl   _T)bounddefaultF)r3   rh   rF   rG   r3   rh   rF   rG   r$   c          
      O   sj   t | g|R |||d|\}} }}|tkrt|}n|}|j| g|R i |}	|	js3td |	S )zHGets a tokenizer for the given model name via HuggingFace or ModelScope.)rh   rF   rG   ziUsing a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead.)ri   r   TokenizerRegistryr/   r0   is_fastr%   r&   )
rE   r3   rh   rF   rG   r1   r2   r!   tokenizer_cls_rj   r)   r)   r*   get_tokenizer   s(   
	rs   model_configc                 K   s.   | j rd S t| jf| j| j| j| jd|S rg   )skip_tokenizer_initcached_get_tokenizerrj   rD   r!   rk   rh   )rt   r2   r)   r)   r*   cached_tokenizer_from_config   s   rw   )rf   r   )rt   r   )1dataclassesr   r   	functoolsr   pathlibr   typingr   r^   typing_extensionsr   r   	vllm.envsrY   vllm.loggerr	   "vllm.transformers_utils.gguf_utilsr
   r   r   r   r   "vllm.transformers_utils.repo_utilsr   r   vllm.utils.import_utilsr   protocolr   vllm.config.modelr   r   r5   r%   _VLLM_TOKENIZERSr   itemsrp   r9   re   ri   rl   rm   r<   boolrs   rv   rw   r)   r)   r)   r*   <module>   st    
\

"