o Gi04@snddlZddlZddlZddlmZddlmZddlmZm Z ddl m Z GdddZ Gdd d eZ dS) N)SentencePieceProcessor)PreTrainedTokenizer) BatchEncoding EncodedInput)PaddingStrategyc @seZdZdefddZddefddZddeded ed eefd d Z d eed efddZ deed efddZ ddZ ddZ dS) SPTokenizer model_pathcCstj|s J|t|d|_|j|_|j|_|j|_|j |_ |j|j ks4Jgd}gd|}i|_ i|_ |D]}|j|j |<||j |j<|jd7_qFddd|D|_dS)N) model_file)z <|system|>z<|user|> <|assistant|>z<|observation|>)z[MASK][gMASK]z[sMASK]sopeop|cSsg|]}t|qS)reescape).0tokenrrX/home/ubuntu/.local/lib/python3.10/site-packages/diffusers/pipelines/kolors/tokenizer.py .sz(SPTokenizer.__init__..)ospathisfilersp_model vocab_sizen_wordsbos_ideos_idunk_idpad_idget_piece_sizespecial_tokensindex_special_tokensjoinrole_special_token_expression)selfrrole_special_tokensr"rrrr__init__s         zSPTokenizer.__init__Fsc Cs|rKd}g}t|j|D](}||kr$||j||||||||}q |t |krI||j||d|S|j|S)Nr) rfinditerr%startextendrEncodeAsPiecesappendendlen)r&r)encode_special_tokens last_indextmatchrrrtokenize0s    zSPTokenizer.tokenizeboseosreturncCs>t|tsJ|j|}|r|jg|}|r||jg}|SN) isinstancestrrencoderr)r&r)r6r7r3rrrr<?s   zSPTokenizer.encoder3cCshdg}}|D] }||jvr"|r||j|7}g}||j|7}q||q|r2||j|7}|S)N)r#rdecoder.)r&r3textbufferrrrrr>Hs   zSPTokenizer.decodetokenscCs|j|}|Sr9)r DecodePieces)r&rAr?rrr decode_tokensVs zSPTokenizer.decode_tokenscCs ||jvr |j|S|j|Sz0Converts a token (str) in an id using the vocab.)r"r PieceToIdr&rrrrconvert_token_to_idZs   zSPTokenizer.convert_token_to_idcCs@||jvr |j|S||j|j|jfvs|dkrdS|j|S)=Converts an index (integer) in a token (str) using the vocab.rr=)r#rrr r IdToPiecer&indexrrrconvert_id_to_token`s   zSPTokenizer.convert_id_to_tokenN)F)FF)__name__ __module__ __qualname__r;r(r5boollistintr<r>rCrGrLrrrrrs   rcseZdZddiZgdZ   d<fdd Zdd Zed efd d Z e j d efdd Z ed efddZ e j d efddZ eddZ ed efddZ e j d efddZ eddZeddZddZddZdd Zd!d"Zd#eed efd$d%Zd=d'd(Zd)d*Zd+d,Zd>d.d/Z &d=d0eed1eed&Bd eefd2d3Zd&ejd&d&d&fd4eeefe Bd5ed&Bd6ed7ed&Bd8e!d&Bd9e!d&Bd efd:d;Z"Z#S)?ChatGLMTokenizer vocab_fileztokenizer.model) input_idsattention_mask position_idsleftFc sTd|_||_t||_|jj|jj|jjd|_||_t j d|||d|dS)N GLMTokenizer)z) padding_sideclean_up_tokenization_spacesr1r) namerTr tokenizerrrr r"r1superr()r&rTr\r]r1kwargs __class__rrr(ns  zChatGLMTokenizer.__init__cCs@||jvr |j|S||jjvsJ|d|j|jj|S)Nz is not a special token for )r"r_r^rFrrr get_commands    zChatGLMTokenizer.get_commandr8cCdSNzrr&rrr unk_tokenzChatGLMTokenizer.unk_tokenvaluecC ||_dSr9) _unk_tokenr&rjrrrrh cCrerfrrgrrr pad_tokenrizChatGLMTokenizer.pad_tokencCrkr9) _pad_tokenrmrrrrorncC |dS)Nr[rdrgrrr pad_token_idrnzChatGLMTokenizer.pad_token_idcCre)Nzrrgrrr eos_tokenrizChatGLMTokenizer.eos_tokencCrkr9) _eos_tokenrmrrrrtrncCrq)NrZrrrgrrr eos_token_idrnzChatGLMTokenizer.eos_token_idcCs|jjSr9)r_rrgrrrrszChatGLMTokenizer.vocab_sizecs(fddtjD}|j|S)zReturns vocab as a dictcsi|]}||qSr)_convert_id_to_token)rirgrr sz.ChatGLMTokenizer.get_vocab..)rangerupdateadded_tokens_encoder)r&vocabrrgr get_vocabs zChatGLMTokenizer.get_vocabcKs|jj||jdS)N)r1)r_r5r1)r&r?rarrr _tokenizeszChatGLMTokenizer._tokenizecC |j|SrD)r_rGrFrrr_convert_token_to_id z%ChatGLMTokenizer._convert_token_to_idcCr)rH)r_rLrJrrrrwrz%ChatGLMTokenizer._convert_id_to_tokenrAcCrr9)r_rC)r&rArrrconvert_tokens_to_strings z)ChatGLMTokenizer.convert_tokens_to_stringNcCstj|rtj||jd}n|}t|jd }|}Wdn1s(wYt|d}||Wd|fS1sDwY|fS)a Save the vocabulary and special tokens file to a directory. Args: save_directory (`str`): The directory in which to save the vocabulary. filename_prefix (`str`, *optional*): An optional prefix to add to the named of the saved files. Returns: `tuple(str)`: Paths to the files saved. rTrbNwb) rrisdirr$vocab_files_namesopenrTreadwrite)r&save_directoryfilename_prefixrTfin proto_strwriterrrrsave_vocabularys     z ChatGLMTokenizer.save_vocabularycCs|d|dg}|S)Nr r rr)r& prefix_tokensrrrget_prefix_tokenssz"ChatGLMTokenizer.get_prefix_tokenscCsN|dvsJ||d|dg|j|d}|j|}||}|S)N)systemuser assistant observationz<|z|> )rdr_r<)r&rolemetadatamessage role_tokensmessage_tokensrArrrbuild_single_messages & z%ChatGLMTokenizer.build_single_messagerc Cs|durg}g}|D].}|d}|ddkr(d|vr(|dtj|dddd}|||d|d d |q |||d |||d g|j|gd d dS)NcontentrrtoolsrF)indent ensure_asciirr=r ptT)return_tensorsis_split_into_words)jsondumpsr,rgetrdbatch_encode_plus)r&queryhistoryrrUitemrrrrbuild_chat_inputs"z!ChatGLMTokenizer.build_chat_input token_ids_0 token_ids_1cCs0|}||}|dur|||dg}|S)a Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A BERT sequence has the following format: - single sequence: `[CLS] X [SEP]` - pair of sequences: `[CLS] A [SEP] B [SEP]` Args: token_ids_0 (`list[int]`): list of IDs to which the special tokens will be added. token_ids_1 (`list[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: `list[int]`: list of [input IDs](../glossary#input-ids) with the appropriate special tokens. NrZ)rrd)r&rrrrrr build_inputs_with_special_tokenss z1ChatGLMTokenizer.build_inputs_with_special_tokensencoded_inputs max_lengthpadding_strategypad_to_multiple_ofreturn_attention_maskr\c Cs|jdksJ||jd}t|}|tjkrt|}|dur1|dur1||dkr1||d|}|tjko;t||k} d|vrGdg||d<d|vrStt||d<| r|t|} d|vrjdg| |d|d<d|vrydg| |d|d<|jg| |||jd<|S)a7 Pad encoded inputs (on left/right and up to predefined length or max length in the batch) Args: encoded_inputs: Dictionary of tokenized inputs (`list[int]`) or batch of tokenized inputs (`list[list[int]]`). max_length: maximum length of the returned list and optionally padding length (see below). Will truncate by taking into account the special tokens. padding_strategy: PaddingStrategy to use for padding. - PaddingStrategy.LONGEST Pad to the longest sequence in the batch - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) - PaddingStrategy.DO_NOT_PAD: Do not pad The tokenizer padding sides are defined in self.padding_side: - 'left': pads on the left of the sequences - 'right': pads on the right of the sequences pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta). padding_side (`str`, *optional*): The side on which the model should have padding applied. Should be selected between ['right', 'left']. Default value is picked from the class attribute of the same name. return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics) rXrNrrVrW) r\model_input_namesr0rLONGEST DO_NOT_PADrQrzrs) r&rrrrrr\required_input seq_lengthneeds_to_be_padded differencerrr_pads($  zChatGLMTokenizer._pad)rXFFr9)Nr)$rMrNrOrrr(rdpropertyr;rhsetterrorsrtrvrr~rrrwrQrrrrrrRrrrdictrrrPr __classcell__rrrbrrSis~       rS)rrr sentencepiecer transformersr$transformers.tokenization_utils_baserrtransformers.utilsrrrSrrrrs   P