o ߥiD@sdZddlZddlmZmZmZmZddlZddl Z ddl m Z ddl mZmZddlmZddlmZeZdd iZGd d d ZGd d d ZGddde ZdS)z!Tokenization classes for ChatGLM.N)DictListOptionalUnion)PreTrainedTokenizer) BatchEncoding EncodedInput)PaddingStrategy)loggerzTHUDM/chatglm-6bic@sVeZdZddZddZdeefddZdd Zd d Z d d Z ddZ ddZ dS) TextTokenizercCs&t|_|j||j|_dSN)spmSentencePieceProcessorspLoad vocab_size num_tokens)self model_pathr^/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/chatglm/tokenization.py__init__s  zTextTokenizer.__init__cC |j|Sr )r EncodeAsIdsrtextrrrencode zTextTokenizer.encodeidscCrr )r DecodeIds)rrrrrdecoderzTextTokenizer.decodecCrr )rEncodeAsPiecesrrrrtokenize!rzTextTokenizer.tokenizecsfdd|DS)Ncsg|]}j|qSrr PieceToId).0tokenrrr %z7TextTokenizer.convert_tokens_to_ids..r)rtokensrr'rconvert_tokens_to_ids$sz#TextTokenizer.convert_tokens_to_idscCrr r#rr&rrrconvert_token_to_id'rz!TextTokenizer.convert_token_to_idcCrr )r IdToPiece)ridxrrrconvert_id_to_token*rz!TextTokenizer.convert_id_to_tokencC|jSr )rr'rrr__len__-zTextTokenizer.__len__N) __name__ __module__ __qualname__rrrintr r"r+r-r0r2rrrrr s r c@seZdZ   d#ddZddZedefd d Zed d Ze d dZ e ddZ ed$de defddZ d%de fddZ   d&de deefddZdeede fddZ   d&de dee fddZdeee ffd d!Zd"S)' SPTokenizer NPTcCs<|dusJ||_||_gd|_||_||_t||_dS)N)[MASK][gMASK]z[sMASK]z zz) vocab_filenum_image_tokensspecial_tokensmax_blank_length byte_fallbackr text_tokenizer)rr?r@rBrCrrrr3s  zSPTokenizer.__init__cCr1r )rDr'rrr_get_text_tokenizerEr3zSPTokenizer._get_text_tokenizerlengthcCs|dksJd|dS)Nz<|blank_z|>r)rFrrrget_blank_tokenHs  zSPTokenizer.get_blank_tokencCsdS)Nz<|tab|>rrrrr get_tab_tokenMszSPTokenizer.get_tab_tokencC|jjSr )rDrr'rrrnum_text_tokensQszSPTokenizer.num_text_tokenscCs |j|jSr )r@rKr'rrrrUs zSPTokenizer.num_tokensrmax_lencCs<|dt}t|ddD] }|d|t|}q|S)N  )replacer8rIrangerH)rrLirrr_encode_whitespacesYszSPTokenizer._encode_whitespacescCs(|r|dd}|r|j||jd}|S)N )rL)rQrTrB)rr linebreak whitespacesrrr _preprocess`s zSPTokenizer._preprocessreturncsN|||}|s d|}|}fdd|D}|r!|S|ddS) @param text: Text to encode. @param linebreak: Whether to encode newline ( ) in text. @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding. @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text. @param add_dummy_prefix: Whether to add dummy blank space in the beginning. rVcsg|]}|jqSrr@)r%xr'rrr(xsz&SPTokenizer.encode..rGN)rYrEr)rrrWrXadd_dummy_prefixtmpr*rr'rrhs  zSPTokenizer.encodetext_idscszfdd|D}dd|D}|}|dd}|td}tdjdD] }||d |}q-|S) Ncsg|] }t|jqSr)r7r@r%_idr'rrr(|sz&SPTokenizer.decode..cSsg|]}|dkr|qS)rrrarrrr(}r)rVrUrMrGrNrP)rEr rQr8rIrRrBrH)rr`rrrSrr'rr {s zSPTokenizer.decodecCs<||||}|s d|}||}|r|S|ddS)r[rVrGN)rYrEr")rrrWrXr^r*rrrr"s  zSPTokenizer.tokenizer]cCst|tr||jkrd|S|j||jSt|tr@|dr7|dr7|dd r7t|ddS|j ||jSt d)Nz zrOzThe key should be str or int.) isinstancer7r@formatrDr0str startswithendswithisdigitr- ValueError)rr]rrr __getitem__s&    zSPTokenizer.__getitem__N)r9r:T)r:)TT)TTT)r4r5r6rrE staticmethodr7rHrIpropertyrKrrgrTrYrrr r"rrlrrrrr81sF        r8c sVeZdZdZddiZeZgdZ      d4 d5fdd Ze d e e fddZ e d e e fddZ e ddZddZddZddZ  d6dee ee fded ed efd!d"Zd#d$Zd%d&Zd7d'd(Z d7d)ee d*e ee d ee fd+d,Zdejddfd-eeeefefd.e e d/ed0e e d1e ed e f d2d3Z!Z"S)8ChatGLMTokenizera Construct a ChatGLM tokenizer. Based on byte-level Byte-Pair-Encoding. Args: vocab_file: Path to the vocabulary file. do_lower_case: Use lower case letters. remove_space: Remove spaces. bos_token: The bos token eos_token: The Eos Token end_token: The end token mask_token: The mask token gmask_token: The gmask token padding_side: The padding side num_image_tokens: The `num_image_tokens` in `SPTokenizer` r?zice_text.model) input_idsattention_mask position_idsFr=r>r;r<leftr9rZNc sht|| d|_tjd||| |||||| d | ||_||_||_||_||_||_ ||_ ||_ dS)Nr\) do_lower_case remove_space padding_side bos_token eos_token end_token mask_token gmask_tokenr@r) r8 sp_tokenizersuperrrurvr?rxryrzr{r|) rr?rurvrxryrzr{r|rwr@kwargs __class__rrrs2   zChatGLMTokenizer.__init__cCs|jdurdS||jSr )r|r+r'rrrgmask_token_ids  zChatGLMTokenizer.gmask_token_idcCs|jdurdS||jS)z `Optional[int]`: Id of the end of context token in the vocabulary. Returns `None` if the token has not been set. N)rzr+r'rrr end_token_ids  zChatGLMTokenizer.end_token_idcCrJ)z Returns vocab size )r}rr'rrrrszChatGLMTokenizer.vocab_sizecs(fddtjD}|j|S)z Returns vocab as a dict csi|]}||qSr)_convert_id_to_token)r%rSr'rr s z.ChatGLMTokenizer.get_vocab..)rRrupdateadded_tokens_encoder)rvocabrr'r get_vocabs  zChatGLMTokenizer.get_vocabcCs0|jr d|}n|}|jr|}|S)NrP)rvjoinstripsplitrulower)rinputsoutputsrrrpreprocess_texts z ChatGLMTokenizer.preprocess_textcKs||}|j|}|S)z Returns a tokenized string. )rr}r")rrrseqrrr _tokenizes  zChatGLMTokenizer._tokenizeT token_idsskip_special_tokensclean_up_tokenization_spacescKsHt|tr|g}t|dkrdS|j|vrtt|jj|}|j|S)Nr) rer7len pad_token_idlistfilter__ne__r}r )rrrrrrrr_decodes    zChatGLMTokenizer._decodecC |j|S)z2 Converts a token (str) in an id using the vocab. r}r,rrr_convert_token_to_id# z%ChatGLMTokenizer._convert_token_to_idcCr)z=Converts an index (integer) in a token (str) using the vocab.r)rindexrrrr'rz%ChatGLMTokenizer._convert_id_to_tokencCstj|rtj||jd}n|}t|jd }|}Wdn1s(wYt|d}||Wd|fS1sDwY|fS)a Save the vocabulary and special tokens file to a directory. Args: save_directory (`str`): The directory in which to save the vocabulary. filename_prefix (`str`, *optional*): An optional prefix to add to the named of the saved files. Returns: `Tuple(str)`: Paths to the files saved. r?rbNwb) ospathisdirrvocab_files_namesopenr?readwrite)rsave_directoryfilename_prefixr?fin proto_strwriterrrrsave_vocabulary+s     z ChatGLMTokenizer.save_vocabulary token_ids_0 token_ids_1cCs|j|j}|j|j}|j|j}||vr||vr||g7}|d|kr4|d|kr4||j|jg7}||j|jg7}|durR|rI|d|krN||g7}||7}|S)a Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A BERT sequence has the following format: - single sequence: `[CLS] X [SEP]` - pair of sequences: `[CLS] A [SEP] B [SEP]` Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. rON)r}r{r|ryrzrx)rrrmask_ids gmask_idseos_idrrr build_inputs_with_special_tokensFs     z1ChatGLMTokenizer.build_inputs_with_special_tokensencoded_inputs max_lengthpadding_strategypad_to_multiple_ofreturn_attention_maskcCsb|j|j}|j|j}|j|j}|jdksJ||jd} t| } |tjkr-t| }|durC|durC||dkrC||d|}|tj koMt| |k} |durd|vr|| vr`| |} n| } t d| | f} t | } d| ddddd| f<t | dk} | |d<d|vrt j| t jd}|| vr|n|}|| vr| |}||| d<t t j| t jdt jd| | dt jdg}t j||gdd |d<| r/|t| }d|vrt j|dd |df|dfgd d d |d<d|vr|jg||d|d<d|vrdg||d|d<d|vr"t j|dd |dfgd|d<|jg|| ||jd<|S)a? Pad encoded inputs (on left/right and up to predefined length or max length in the batch) Args: encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). max_length: maximum length of the returned list and optionally padding length (see below). Will truncate by taking into account the special tokens. padding_strategy: PaddingStrategy to use for padding. - PaddingStrategy.LONGEST Pad to the longest sequence in the batch - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) - PaddingStrategy.DO_NOT_PAD: Do not pad The tokenizer padding sides are defined in self.padding_side: - 'left': pads on the left of the sequences - 'right': pads on the right of the sequences pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta). return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics) rtrNrNrqg?rr)dtype)axis)rrconstantT) pad_widthmodeconstant_valuestoken_type_idsspecial_tokens_mask)r)r}rxr{r|rwmodel_input_namesrr LONGEST DO_NOT_PADrnponestrilbool_arangeint64 concatenatezerosstackpadpad_token_type_idr)rrrrrr bos_token_id mask_token_idrrequired_input seq_lengthneeds_to_be_paddedcontext_lengthrqrrr{ mask_positionblock_position_ids differencerrr_padls                 zChatGLMTokenizer._pad) FFr=r>rsr;r<rtr9)rZN)FTr )#r4r5r6__doc__r&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESmax_model_input_sizesrrrnrr7rrrrrrrrboolrgrrrrrr rrrrdictr __classcell__rrrrros '         )ro)rrtypingrrrrnumpyr sentencepiecer transformers.tokenization_utilsr$transformers.tokenization_utils_baserrtransformers.utilsr modelscope.utilsr logging get_loggerrr r8rorrrrs   x