o *i)@s ddlZddlZddlZddlmZddlmZddlZddl m Z ddl m Z m Z ddlmZmZmZddlmZmZejded d ejded d e rUdd lmZd eeBdefddZddeeBdedefddZdeeBdedBfddZGdddeZdS)N)cached_property)Path)TokenizerException)assert_sentencepiece_installedis_sentencepiece_installed)SpecialTokenPolicy TokenizerTokenizerVersion) ImageConfigMultiModalVersiononcez%.*`get_control_token` is deprecated.*)actioncategorymessagez#.*`_control_tokens` is deprecated.*)SentencePieceProcessorpathreturncsbttr tttj}ttjdgfdd|Ddg}o0tfdd|DS)z1Check if the given path is a SentencePiece model.cs$g|]}D] }d||qqS)z.model.).0vm) mm_versionsrk/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/sentencepiece.py )s$z$is_sentencepiece..z.modelc3s|] }j|VqdSN)nameendswith)rsuffix)rrr +sz#is_sentencepiece..) isinstancestrrlistr __members__r is_fileany)rinstruct_versionssuffixesr)rrris_sentencepiece"s  r(Ftokenizer_filenameraise_deprecatedcCspt|}|dd}|dkr|dd}|dkr(|r$td|dtdS|tjvr4td |t|S) z3Get the version of the tokenizer from the filename..modelrrz4Make sure to rename your tokenizer file to end with z.v1.v1!Unrecognized tokenizer filename: )r!splitrr r#)r)r* _version_strrrrget_spm_version.s r2cCs^t|}|dd}|dksd|vrdSd|dd}|tjvr*td|t|jS)z1Get the image config from the tokenizer filename.r+r,r-rNr/)r!r0r r#rconfig)r)r1_mm_version_strrrrget_image_configCs  r5c seZdZdZd7deeBdedBddffdd Zedefdd Z edefd d Z d ede fd dZ d ede fddZ ede fddZede fddZdeefddZede fddZede fddZde ejBeBdefddZedee fddZedee fd d!Zd ed"ed#edee fd$d%Zd7d&ee d'edBdefd(d)Zd*e defd+d,Zd&ee d'edefd-d.Z d&ee defd/d0Z!d&ee defd1d2Z"ede fd3d4Z#ede fd5d6Z$Z%S)8SentencePieceTokenizerzC[SentencePiece](https://github.com/google/sentencepiece) tokenizer.N model_pathtokenizer_versionrcsttjj_tj|sJ|t t |t r|n| d_ j j ks0JfddtjD_|pDt|dd_t|_tdS)zInitialize the `SentencePieceTokenizer`. Args: model_path: The path to the `SentencePiece` model. tokenizer_version: The version of the tokenizer. If not provided, it will be inferred from the model path. ) model_filecsg|]}j|qSr_model id_to_piece)riselfrrrgsz3SentencePieceTokenizer.__init__..F)r*N)rlogging getLogger __class____name___loggerosrisfilerr r!as_posixr; vocab_sizeget_piece_sizerangen_words_vocabr2_versionr _file_pathsuper__init__)r?r7r8rBr>rrPVs zSentencePieceTokenizer.__init__cC|jS)z The path to the tokenizer model.)rNr>rrr file_pathnz SentencePieceTokenizer.file_pathcCrR)zThe version of the tokenizer.)rMr>rrrversionsrTzSentencePieceTokenizer.versionscC |j|S)z+Get the special token for the given string.)r; piece_to_idr?rVrrrget_special_tokenx z(SentencePieceTokenizer.get_special_tokencCtdt||S)NzC`get_control_token` is deprecated. Use `get_special_token` instead.)warningswarn FutureWarningrZrYrrrget_control_token|s  z(SentencePieceTokenizer.get_control_tokencC |jS)z!Vocabulary size of the tokenizer.)r;rHr>rrrrK zSentencePieceTokenizer.n_wordscCs t|jS)z.The number of special tokens of the tokenizer.)len special_idsr>rrrnum_special_tokensrbz)SentencePieceTokenizer.num_special_tokenscCrR)z,Get all tokens in the vocabulary as strings.)rLr>rrrvocabszSentencePieceTokenizer.vocabcCra)z#The beginning of sentence token id.)r;bos_idr>rrrrgrbzSentencePieceTokenizer.bos_idcCra)zThe end of sentence token id.)r;eos_idr>rrrrhrbzSentencePieceTokenizer.eos_idtokencCsVt|ttjfr|jt|St|tr!|j|}|j|Stdt |j )z7Return `True` if the passed `token` is a special token.zExpected int or str, got ) r intnpintegerr; IsControlr!rX TypeErrortyperC)r?ri token_intrrr is_specials    z!SentencePieceTokenizer.is_specialcCstdt|jS)NzS`_control_tokens` is deprecated. Make use of `is_special` or `special_ids` instead.)r]r^r_rdr>rrr_control_tokenssz&SentencePieceTokenizer._control_tokenscsfddtjDS)zIds of the special tokens.csh|] }j|r|qSr)r;rmrtokr>rr sz5SentencePieceTokenizer.special_ids..)rJrKr>rr>rrdsz"SentencePieceTokenizer.special_idsboseoscCs@t|tsJ|j|}|r|jg|}|rg||j}|S)aEncode the given string into a list of token ids. Args: s: The string to encode. bos: Whether to add the beginning of sentence token. eos: Whether to add the end of sentence token. Returns: The list of token ids. )r r!r;encodergrh)r?rVrvrwtrrrrxs  zSentencePieceTokenizer.encodetokensspecial_token_policycCsl|durt|ttfstdt|d|dur"tdttj}|tj tj fvr0| ||S|j |S)aaDecode the given list of token ids into a string. Note: Using `special_token_policy=SpecialTokenPolicy.KEEP` will keep the special tokens and the normal tokens as SentencePiece pieces. Args: tokens: The list of token ids. special_token_policy: The policy to use for special tokens. If `None`, the default policy is `SpecialTokenPolicy.IGNORE`. Passing `None` is deprecated and will be changed to `SpecialTokenPolicy.IGNORE` in `mistral_common=1.10.0`. Returns: The decoded string. NzFExpected `special_token_policy` to be None or SpecialTokenPolicy, got r+zUsing the tokenizer's special token policy `None` is deprecated. It will be removed in 1.10.0. Please pass a special token policy explicitly. Future default will be SpecialTokenPolicy.IGNORE.)r r!r ValueErrorror]r^r_IGNOREKEEPRAISE_decode_with_special_tokensr;decode)r?rzr{rrrrs   zSentencePieceTokenizer.decodetoken_idcCrW)z,Convert the given token id to a token piece.r:)r?rrrrr<r[z"SentencePieceTokenizer.id_to_piececsg}g}|D].}|r/|tjkrtd|r&|fdd|Dg}||q||q|rC|fdd|Dd|S)NzNDecoding `tokens` that contain special tokens with special_token_policy=RAISE.cg|]}|qSrr<rsr>rrrzFSentencePieceTokenizer._decode_with_special_tokens..crrrrsr>rrrrr)rqrrr|extendappendr<join)r?rzr{ text_list curr_tokensrtrr>rrs    z2SentencePieceTokenizer._decode_with_special_tokenscCr\)z[DEPRECATED] Converts a list of token ids into a string, keeping special tokens. Use `decode` with `special_token_policy=SpecialTokenPolicy.KEEP` instead. This is a convenient method for debugging. z`to_string` is deprecated and will be removed in 1.10.0. Use `decode` with `special_token_policy=SpecialTokenPolicy.KEEP` instead.)r]r^r_ _to_stringr?rzrrr to_strings  z SentencePieceTokenizer.to_stringcCs|j|tjdS)N)r{)rrr~rrrrrsz!SentencePieceTokenizer._to_stringcCra)zThe padding token id.)r;pad_idr>rrrrrbzSentencePieceTokenizer.pad_idcCra)zThe unknown token id.)r;unk_idr>rrrrrbzSentencePieceTokenizer.unk_idr)&rC __module__ __qualname____doc__r!rr rPpropertyrSrUrjrZr`rKrer"rfrrgrhrkrlboolrqsetrrrdrxrrr<rrrrr __classcell__rrrQrr6SsB$  &r6)F)r@rEr] functoolsrpathlibrnumpyrkmistral_common.exceptionsrmistral_common.importsrr%mistral_common.tokens.tokenizers.baserrr &mistral_common.tokens.tokenizers.imager r filterwarningsr_ sentencepiecerr!rr(r2r5r6rrrrs4