o ߥi"@sdZddlmZmZmZmZddlZddlZddlZddl Z ddl m Z ddl m Z mZmZmZddlZddlZddlmZmZddlmZeZdd iZGd d d eZdS) zTokenization classes for QWen.)absolute_importdivisionprint_functionunicode_literalsN)open)ListOptionalTupleUnion) AddedTokenPreTrainedTokenizer) get_logger vocab_file qwen.tiktokencseZdZdZ eZ         d+fdd Zd d Zd d Zd dZ de de e fddZ de de e fddZde e de fddZeddZdede fddZde defdd Zede e fd!d"Zede efd#d$Zd%d&Z d,d'eee efd(ede fd)d*ZZS)- QWenTokenizerzQWen tokenizer.replaceN <|endoftext|>FTc  st|tr t|dddn|}t|trt|dddn|}t|tr(t|dddn|}t|tr6t|dddn|}tj||||||| d| |_|durN|ntd|_||_d} d} d}d}| rv| ||d d d d d ft ddt dD}n| ||f}d}dtddfdd}||}ddt |t |dD}||_ tj| |||d}t |t ||jksJt |t |d|jd||_|j|_dd|jD|_||_|jj|_|||_|||_dS)NF)lstriprstrip)errors unk_token bos_token eos_token pad_tokenadd_prefix_space add_bos_tokengmBQwenrz <|im_start|>z <|im_end|>zzzzzcSsg|]}d|dqS)z.0irr[/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/qwen/tokenization.py Wsz*QWenTokenizer.__init__..zn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+tiktoken_bpe_filereturnzdict[bytes, int]cSs*t|d}dddd|DDS)NrbcSs i|] \}}t|t|qSr)base64 b64decodeint)r tokenrankrrr" aszEQWenTokenizer.__init__..load_tiktoken_bpe..css|] }|r|VqdSN)split)r linerrr" cszDQWenTokenizer.__init__..load_tiktoken_bpe..)rread splitlines)r%contentsrrr"load_tiktoken_bpe_s z1QWenTokenizer.__init__..load_tiktoken_bpecSi|]\}}||qSrr)r indexr+rrr"r-hsz*QWenTokenizer.__init__..)start)pat_strmergeable_ranksspecial_tokensz != z in encodingcSr6rr)r kvrrr"r-z) isinstancestrr super__init__rr*max_lenrtuplerange enumeratelenr;tiktokenEncodingn_vocabr:encoderitemsdecoder tokenizer eot_tokeneod_id im_start_id im_end_id)selfrrrCrrrrrradd_more_sp_tokenskwargsname ENDOFTEXTIMSTARTIMENDr;PAT_STRr5r:enc __class__rr"rB!s     zQWenTokenizer.__init__cC|jjSr.rNrJrSrrr"__len__szQWenTokenizer.__len__cCs|jSr.)r:r`rrr" get_vocabszQWenTokenizer.get_vocabcCsg}t|tr||jvr|j|S|j|S|D]}||jvr)||j|q||j|qt||jkrFt d t||j|S)NzToken indices sequence length is longer than the specified maximum sequence length for this model ({} > {}). Running this sequence through the model will result in indexing errors) r?r@r;rKgetappendrGrCloggerwarningformat)rStokensidsr+rrr"convert_tokens_to_idss     z#QWenTokenizer.convert_tokens_to_idssave_directoryr&cKstj|d}t|ddd)}|jD]\}}t|ddt |d}| |qWd|fS1s:wY|fS)z Save only the vocabulary of the tokenizer (vocabulary + added tokens). Returns: `Tuple(str)`: Paths to the files saved. rwutf8)encoding  N) ospathjoinrr:rLr( b64encodedecoder@write)rSrkrU file_pathrlr<r=r0rrr"save_vocabularys   zQWenTokenizer.save_vocabularytextcKs6g}td|}|j|D] }||j|q|S)a Converts a string in a sequence of tokens, replacing unknown tokens with the `unk_token`. Args: text (`str`): The sequence to be encoded. kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific encode method. See details in [`~PreTrainedTokenizerBase.__call__`] Returns: `List[str]`: The list of tokens. NFC) unicodedata normalizerNencode_ordinaryrdrM)rSryrUrhtrrr"tokenizes  zQWenTokenizer.tokenizerhcs0d|}tfdd|Djdjd}|S)z Converts a sequence of tokens in a single string. The most simple way to do it is `" ".join(tokens)` but we often want to remove sub-word tokenization artifacts at the same time. csg|]}j|qSr) byte_decoder)r cr`rr"r#r>z:QWenTokenizer.convert_tokens_to_string..zutf-8)r)rs bytearrayrur)rSrhryrr`r"convert_tokens_to_strings z&QWenTokenizer.convert_tokens_to_stringcCr^r.r_r`rrr" vocab_sizeszQWenTokenizer.vocab_sizer7cCs ||jjkr |jS|j|gSr.)rNrJrru)rSr7rrr"_convert_id_to_tokens z"QWenTokenizer._convert_id_to_tokenr+cCs&|j|d|jj|jdddS)z*Converts a token to an id using the vocab.zUTF-8all)allowed_specialr)rKrcencoderNr)rSr+rrr"_convert_token_to_idsz"QWenTokenizer._convert_token_to_idcCdd|jD}|S)z `List[str]`: All the special tokens (`''`, `''`, etc.) mapped to class attributes. Convert tokens of `tokenizers.AddedToken` type to string. cSsg|]}t|qSr)r@)r srrr"r#sz4QWenTokenizer.all_special_tokens..)r;keys)rSall_toksrrr"all_special_tokenssz QWenTokenizer.all_special_tokenscCr)zy `List[int]`: List the ids of the special tokens(`''`, `''`, etc.) mapped to class attributes. cSsg|]}|qSrr)r r=rrr"r#sz1QWenTokenizer.all_special_ids..)r;values)rSall_idsrrr"all_special_idsszQWenTokenizer.all_special_idscKst)a Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). Do NOT take care of added tokens. )NotImplementedError)rSryrUrrr" _tokenizeszQWenTokenizer._tokenize token_idsskip_special_tokensc s2t|tr|g}|rfdd|D}j|S)Ncsg|] }|jvr|qSr)rrr`rr"r#sz)QWenTokenizer._decode..)r?r*rNru)rSrrrUrr`r"_decodes  zQWenTokenizer._decode) rNrrrNFFT)F)__name__ __module__ __qualname____doc__VOCAB_FILES_NAMESvocab_files_namesrBrarbrjr@r rxrrrpropertyrr*rrrrrr boolr __classcell__rrr\r"rsJ_   r)r __future__rrrrr(loggingrqr{iortypingrrr r jsonrH transformersr r modelscope.utils.loggerr rerrrrrr"s