o c۷i^@sXUddlZddlZddlmZddlmZddlZddlmZddl m Z e e d<z ddl Z ddlZ dZWn ey@dZYnweeZdd d Zd eed ed eeefddZdededBd eefddZdeed eeeeffddZdeed efddZ     d dedeeBdBde eBdBdedBde de d efddZdS)!N)Path)Iterator)TokenizerVersion)MultiModalVersion_hub_installedTFreturncCststddS)NzxPlease install the `huggingface_hub` package to use this method. Run `pip install mistral-common[hf-hub]` to install it.)r ImportErrorr r \/home/ubuntu/vllm_env/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/utils.py_assert_hub_installeds r lst chunk_sizeccs.tdt||D] }||||Vq dS)aChunk a list into smaller lists of a given size. Args: lst: The list to chunk. chunk_size: The size of each chunk. Returns: An iterator over the chunks. Examples: >>> all_chunks = list(chunks([1, 2, 3, 4, 5], 2)) rN)rangelen)r r ir r r chunkss rrepo_idrevisioncCstttjjtjjdg|d}|dur?|dtjj}| r?| d }| }Wdn1s:wY|rP|d|}| rPt |SgS)zlist the files of a local Hugging Face repo. Args: repo_id: The Hugging Face repo ID. revision: The revision of the model to use. If `None`, the latest revision will be used. models/Nrefsr snapshots)r rhuggingface_hub constants HF_HUB_CACHEREPO_ID_SEPARATORjoinsplitDEFAULT_REVISIONis_fileopenreadis_diroslistdir)rr repo_cache revision_filefile revision_dirr r r list_local_hf_repo_files0s    r*filescsg}ttj}ttjdgfdd|Ddg}|D]&}t|}|j}d|j}|dkr8|||fq||vrC|||fq|S)zFilter the valid tokenizer files from a list of files. Args: files: The list of files to filter. Returns: The list of tuples of file names and paths to the valid tokenizer files. cs$g|]}D] }d||qqS)z.model.r ).0vm mm_versionsr r Xs$z1_filter_valid_tokenizer_files..z.model tekken.json) listr __members__rrnamersuffixesappend)r+valid_tokenizer_filesinstruct_versionssentencepiece_suffixesr( pathlib_file file_namesuffixr r0r _filter_valid_tokenizer_filesKs  r?cCst|}t|dkrtdt|dkr:|D] \}}d|kr"|Sqt|ddddd}td |d |S|dd}|S) zGet one valid tokenizer file from a list of files. Args: files: The list of files to filter. Returns: The path to the tokenizer file. rzNo tokenizer file found.r3cSs|dS)Nrr )xr r r xsz.get_one_valid_tokenizer_file..)keyz,Multiple valid tokenizer files found. Using .)r?r ValueErrorsortedloggerwarning)r+$valid_tokenizer_file_names_and_filesr=tokenizer_filer r r get_one_valid_tokenizer_filefs    rL cache_dirtokenforce_downloadlocal_files_onlyc Cs*t|r |r td|sYzt}|j|||d}d}WnRtjtjtjfyX}z)|r/|t ||d}d}t dt |dkrNt d|d |d |WYd }~nd }~wwt ||d}t |dkrpt d |d |d zt|d} Wntytd|dwtj||| ||||d} | S)aDownload the tokenizer file of a Mistral model from the Hugging Face Hub. See [here](https://huggingface.co/mistralai/models) for a list of our OSS models. Note: You need to install the `huggingface_hub` package to use this method. Please run `pip install mistral-common[hf-hub]` to install it. Args: repo_id: The Hugging Face repo ID. cache_dir: The directory where the tokenizer will be cached. token: The Hugging Face token to use to download the tokenizer. revision: The revision of the model to use. If `None`, the latest revision will be used. force_download: Whether to force the download of the tokenizer. If `True`, the tokenizer will be downloaded even if it is already cached. local_files_only: Whether to only use local files. If `True`, the tokenizer will be downloaded only if it is already cached. Returns: The downloaded tokenizer local path for the given model ID. zSYou cannot force the download of the tokenizer if you only want to use local files.)rrNF)rrTzBCould not connect to the Hugging Face Hub. Using local files only.rzXCould not connect to the Hugging Face Hub and no local files were found for the repo ID z and revision z6. Please check your internet connection and try again.Nz%No local files found for the repo ID zz. Please check the repo ID and the revision or try to download the tokenizer without setting `local_files_only` to `True`.)r+z*No valid tokenizer file found in the repo rE)rrMfilenamerNrrPrO)r rFrHfApilist_repo_filesrequestsConnectionError HTTPErrorTimeoutr*rHinforFileNotFoundErrorrLhf_hub_download) rrMrNrrOrPhf_api repo_fileserKtokenizer_pathr r r download_tokenizer_from_hf_hubsZ        r_)rN)NNNFF)loggingr$pathlibrtypingrrT%mistral_common.tokens.tokenizers.baser&mistral_common.tokens.tokenizers.imagerbool__annotations__rhuggingface_hub.constantsrr getLogger__name__rHr r4strintrr*tupler?rLr_r r r r sR        ""