o c۷iE @s6UddlmZddlmZmZmZddlmZddlm Z ddl m Z m Z m Z mZmZddlmZmZddlmZddlmZmZmZdd lmZdd lmZdd lmZmZm Z dd l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(dd l)m*Z*m+Z+m,Z,ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8ddl9m:Z:m;Z;ddldede:defddZ?Gdddeee ee e'fZ@idddde@jAd d!dd"e@jBd#e@jAd$e@jBd%e@jAd&d'dd(e@jCd)e@jAd*e@jBd+e@jDd,d-dd.e@jBd/e@jBd0d1dZEeFeGege@ffeHd2<d3S)4)Path)AnyCallableGeneric)TokenizerException) FIMRequest)UATSAssistantMessageTypeSystemMessageTypeToolMessageTypeUserMessageType)InstructRequestNormalizerget_normalizer)ChatCompletionRequest)MistralRequestValidatorValidationMode get_validator) SpeechRequest)TranscriptionRequest) AudioConfig AudioEncoderSpecialAudioIDs)InstructRequestInstructRequestTypeInstructTokenizerSpecialTokenPolicy SpecialTokens TokenizedTypeTokenizerVersion) ImageConfig ImageEncoderSpecialImageIDs)InstructTokenizerV1InstructTokenizerV2InstructTokenizerV3InstructTokenizerV7InstructTokenizerV11InstructTokenizerV13InstructTokenizerV15)SentencePieceTokenizerget_image_configis_sentencepiece) Tekkenizer is_tekken)download_tokenizer_from_hf_hub image_config tokenizerreturncCs6t|tjj|tjj|tjjd}t||S)zLoad a image encoder from a config and a tokenizer. Args: image_config: The image config. tokenizer: The tokenizer. Returns: The image encoder. )img img_breakimg_end)r!get_special_tokenrr2valuer3r4r )r/r0 special_idsr8^/home/ubuntu/vllm_env/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/mistral.pyload_image_encoder9s     r: audio_configcs^dtdtdBffdd }t|tjj|tjj|tjj|tjj|tj jd}t ||S)zLoad a audio encoder from a config and a tokenizer. Args: audio_config: The audio config. tokenizer: The tokenizer. Returns: The audio encoder. tokenr1Ncs|sdS|SN) is_specialr5)r<r0r8r9get_special_token_or_noneVs  z5load_audio_encoder..get_special_token_or_none)audio begin_audio streaming_pad text_to_audio audio_to_text) strintrrrAr6rBrCrDrEr)r;r0r@r7r8r?r9load_audio_encoderKs       rHc@seZdZdZdeeeeefde e ee e fde e ee e effddZdeeeedfffd d Zedefd d Zedefd dZedefddZed=ddZed=ddZed>dededdfddZed?deddfddZ ed@de!deddfdd Z"e#d!d!ddej$fd"e!d#ee!Bd!Bd$e!d!Bd%ed&ed'eddfd(d)Z%eej$fd*e!eBd'eddfd+d,Z& !dAd-e'e(d.e)d!Bdefd/d0Z*d-e+defd1d2Z,d-e-defd3d4Z.d-edefd5d6Z/e0j1fd7e2e)d8e0de!fd9d:Z3d7e2e)de!fd;d<Z4d!S)BMistralTokenizeragMistral tokenizer. This class is a wrapper around a [InstructTokenizer][mistral_common.tokens.tokenizers.base.InstructTokenizer], a [MistralRequestValidator][mistral_common.protocol.instruct.validator.MistralRequestValidator] and a [InstructRequestNormalizer][mistral_common.protocol.instruct.normalize.InstructRequestNormalizer]. It provides a convenient interface to tokenize, validate ad normalize Mistral requests. Attributes: instruct_tokenizer: The instruct tokenizer to use. See [InstructTokenizer][mistral_common.tokens.tokenizers.instruct.InstructTokenizer]. instruct_tokenizer validatorrequest_normalizercCs||_||_||_dS)zInitializes a `MistralTokenizer`. Args: instruct_tokenizer: The instruct tokenizer to use. validator: The request validator to use. request_normalizer: The request normalizer to use. N)"_chat_completion_request_validator_instruct_request_normalizerrJ)selfrJrKrLr8r8r9__init__vszMistralTokenizer.__init__r1.cCstj|jjj|jffS)z Provides a recipe for pickling (serializing) this object, which is necessary for use with multiprocessing. Returns: A tuple of the factory function and the arguments to reconstruct the object from its source file. )rI from_filerJr0 file_pathmoderOr8r8r9 __reduce__szMistralTokenizer.__reduce__cCs|jjS)z%The validation mode of the tokenizer.)rMrSrTr8r8r9rSszMistralTokenizer.modecCs |jjjS)zThe version of the tokenizer.)rJr0versionrTr8r8r9rVs zMistralTokenizer.versioncCsttjddS)Ndata)r__file__parentsclsr8r8r9 _data_pathszMistralTokenizer._data_pathcC|jt|dtjdS)zGet the Mistral tokenizer v1.ztokenizer.model.v1rSrQrFr]rtestr[r8r8r9v1szMistralTokenizer.v1cCr^)zGet the Mistral tokenizer v2.z*mistral_instruct_tokenizer_240216.model.v2r_r`r[r8r8r9v2szMistralTokenizer.v2Fr-is_mmcCsL|r|rd}n|r|sd}n |s|rtdd}|jt||tjdS)a;Get the Mistral tokenizer v3. Args: is_tekken: Whether the tokenizer is a tekken tokenizer. See [Tekkenizer][mistral_common.tokens.tokenizers.tekken.Tekkenizer]. is_mm: Whether to load image tokenizer. Returns: The Mistral tokenizer v3. ztekken_240911.jsonztekken_240718.jsonz;Multimodal tokenizer is currently only supported for tekkenz*mistral_instruct_tokenizer_240323.model.v3r_) ValueErrorrQrFr]rra)r\r-rdtokenizer_namer8r8r9v3s zMistralTokenizer.v3cCs<|r|jt|dtjdS|jt|dtjdS)zGet the Mistral tokenizer v7. Args: is_mm: Whether to load the image tokenizer. Returns: The Mistral tokenizer v7. z,mistral_instruct_tokenizer_241114.model.v7m1r_z*mistral_instruct_tokenizer_241114.model.v7r`)r\rdr8r8r9v7s zMistralTokenizer.v7TmodelstrictcCs,|std|tvrtd|t|S)zGet the Mistral tokenizer for a given model. Args: model: The model name. strict: Has to be True, not used. Returns: The Mistral tokenizer for the given model. z&strict has to be `True` since v1.10.0.zUnrecognized model: )reMODEL_NAME_TO_TOKENIZER_CLSr)r\rirjr8r8r9 from_models   zMistralTokenizer.from_modelNrepo_idr<revisionforce_downloadlocal_files_onlyrScCs t|||||d}tj||dS)aODownload the Mistral tokenizer for a given Hugging Face repository ID. See [here](https://huggingface.co/mistralai/models) for a list of our OSS models. Args: repo_id: The Hugging Face repo ID. token: The Hugging Face token to use to download the tokenizer. revision: The revision of the model to use. If `None`, the latest revision will be used. mode: The validation mode to use. force_download: Whether to force the download of the tokenizer. If `True`, the tokenizer will be downloaded even if it is already cached. local_files_only: Whether to only use local files. If `True`, the tokenizer will be downloaded only if it is already cached. Returns: The Mistral tokenizer for the given model. )rmr<rnrorpr_)r.rIrQ)rmr<rnrorprStokenizer_pathr8r8r9 from_hf_hubszMistralTokenizer.from_hf_hubtokenizer_filenamec Cst|rt|}|j}|j}nt|rt|}t|}d}ntd||dur/t ||nd}d}|durEt |ts@Jdt ||}t |j |j}t|j |d} |j tjkrr|dusaJd|dusiJdtt|| |dS|j tjkr|dusJd|dusJdtt|| |dS|j tjkr|dusJdtt||d| |dS|j tjkrtt|||d | |dS|j tjkrtt|||d | |dS|j tjkrtt|||d | |dS|j tjkrtt|||d | |dStd |) zLoads a tokenizer from a file. Args: tokenizer_filename: The path to the tokenizer file. mode: The validation mode to use. Returns: The loaded tokenizer. NzUnrecognized tokenizer file: z-Audio is only supported for tekken tokenizersr_z#Tokenizer version needs to be >= v3z#Tokenizer version needs to be >= v7)rKrL) image_encoder)rt audio_encoderz!Unrecognized tokenizer filename: ) r-r,rQimagerAr+r)r*rr: isinstancerHrrVmodel_settings_builderrrrbrIr"rcr#rgr$rhr%v11r&v13r'v15r() r\rsrSr0r/r;rtrurLrKr8r8r9rQs              zMistralTokenizer.from_filerequestmax_model_input_lencCsF|j|}|dur|jrtd|j|}|jr||_|j|S)aDEncodes a chat completion request. Args: request: The chat completion request to encode. max_model_input_len: The maximum length of the input to the model. If `None`, the input will not be truncated. Returns: The encoded chat completion request. NzUencoding a chat completion request with truncation, but no max model len was provided) rMvalidate_requesttruncate_for_context_lengthrrNfrom_chat_completion_requesttruncate_at_max_tokensrJencode_instruct)rOr|r}validated_requestinstruct_requestr8r8r9encode_chat_completionhs   z'MistralTokenizer.encode_chat_completioncC |j|S)zEncodes a transcription request. Args: request: The transcription request to encode. Returns: The encoded transcription request. )rJencode_transcriptionrOr|r8r8r9r z%MistralTokenizer.encode_transcriptioncCr)zEncodes a speech synthesis request. Args: request: The speech request to encode. Returns: The encoded speech request. )rJencode_speech_requestrr8r8r9rrz&MistralTokenizer.encode_speech_requestcCr)zEncodes a fill in the middle request. Args: request: The fill in the middle request to encode. Returns: The encoded fill in the middle request. )rJ encode_fimrr8r8r9rrzMistralTokenizer.encode_fimtokensspecial_token_policycCs|jj||dS)zDecodes a list of tokens into a string. Args: tokens: The tokens to decode. special_token_policy: The policy to use for special tokens. Returns: The decoded string. )r)rJdecode)rOrrr8r8r9rs zMistralTokenizer.decodecCrr=)rJ _to_string)rOrr8r8r9rs zMistralTokenizer._to_string)r1rI)FF)F)Tr=)5__name__ __module__ __qualname____doc__rrrrr rr r r r rrPtuplerrrUpropertyrrSrrV classmethodrr]rbrcboolrgrhrFrl staticmethodrarrrQrrrGrrrrrrrIGNORElistrrr8r8r8r9rIfs        " Y    rIzministral-8b-2410cC tjddSNT)r-rIrgr8r8r8r9 rzmistral-tiny-2312zopen-mistral-nemo-2407cCrrrr8r8r8r9rrzmistral-tiny-2407zmistral-small-2312zopen-mixtral-8x22b-2404zmistral-small-2402zmistral-small-2409cCrrrr8r8r8r9rrzmistral-medium-2312zmistral-large-2402zmistral-large-2407zmistral-large-2411zpixtral-large-2411cCr)NT)rd)rIrhr8r8r8r9rrzcodestral-2405zcodestral-mamba-2407zpixtral-12b-2409cCstjdddS)NT)r-rdrr8r8r8r9rsrkN)Ipathlibrtypingrrrmistral_common.exceptionsr#mistral_common.protocol.fim.requestr)mistral_common.protocol.instruct.messagesrr r r r *mistral_common.protocol.instruct.normalizer r(mistral_common.protocol.instruct.requestr*mistral_common.protocol.instruct.validatorrrr&mistral_common.protocol.speech.requestr-mistral_common.protocol.transcription.requestr&mistral_common.tokens.tokenizers.audiorrr%mistral_common.tokens.tokenizers.baserrrrrrr&mistral_common.tokens.tokenizers.imagerr r!)mistral_common.tokens.tokenizers.instructr"r#r$r%r&r'r(.mistral_common.tokens.tokenizers.sentencepiecer)r*r+'mistral_common.tokens.tokenizers.tekkenr,r-&mistral_common.tokens.tokenizers.utilsr.r:rHrIrcrgrbrhrkdictrF__annotations__r8r8r8r9sr     $ $   T