o i7@sddlZddlmZddlmZmZmZddlZddl m Z m Z ddl m Z ddlmZer4ddlmZe eZd d iZd Zed d Gddde ZdgZdS)N)copyfile) TYPE_CHECKINGAnyOptional) AddedTokenPreTrainedTokenizer)logging)requires) TextInput vocab_fileztokenizer.modelu▁) sentencepiece)backendsc sXeZdZdZeZddgZ      d4d eee e fffd d Z ddZ ddZ eddZddZdddee ffdd ZddZddZdd Zd!d"Zd5d#ee dee fd$d%Zd5d&d'Z d6d(eed)eeed*edeeffd+d, Z d5d(eed)eeedeefd-d.Z d7d/eed0ed1ede fd2d3ZZS)8GemmaTokenizera Construct a Gemma tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as there is no padding token in the original model. Args: vocab_file (`str`): Path to the vocabulary file. unk_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `""`): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. bos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `""`): The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. eos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `""`): The end of sequence token. pad_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `""`): A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by attention mechanisms or loss computation. sp_model_kwargs (`dict[str, Any]`, `Optional`, *optional*): Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set: - `enable_sampling`: Enable subword regularization. - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout. - `nbest_size = {0,1}`: No sampling is performed. - `nbest_size > 1`: samples from the nbest_size results. - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for BPE-dropout. add_bos_token (`bool`, *optional*, defaults to `True`): Whether or not to add an `bos_token` at the start of sequences. add_eos_token (`bool`, *optional*, defaults to `False`): Whether or not to add an `eos_token` at the end of sequences. clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`): Whether or not to cleanup spaces after decoding, cleanup consists in removing potential artifacts like extra spaces. use_default_system_prompt (`bool`, *optional*, defaults to `False`): Whether or not the default system prompt for Gemma should be used. spaces_between_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to add spaces between special tokens. input_idsattention_maskNTFsp_model_kwargsc s|durin||_t|trt|dddn|}t|tr#t|dddn|}t|tr1t|dddn|}t|tr?t|dddn|}||_||_||_| |_tj di|j|_ |j |t j d|||||||| | | d | dS)NFT) normalizedspecial) bos_token eos_token unk_token pad_token add_bos_token add_eos_tokenrclean_up_tokenization_spacesuse_default_system_promptspaces_between_special_tokens)r isinstancestrrr rrr spmSentencePieceProcessorsp_modelLoadsuper__init__) selfr rrrrrrrrr r!kwargs __class__r"`/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/gemma/tokenization_gemma.pyr*^s2  zGemmaTokenizer.__init__cCs$|j}d|d<|j|d<|S)Nr'sp_model_proto)__dict__copyr'serialized_model_proto)r+stater"r"r/ __getstate__s zGemmaTokenizer.__getstate__cCs2|j|tjdi|j|_|j|jdS)Nr")r1updater%r&rr'LoadFromSerializedProtor0)r+dr"r"r/ __setstate__s zGemmaTokenizer.__setstate__cCs |jS)zReturns vocab size)r'get_piece_sizer+r"r"r/ vocab_sizes zGemmaTokenizer.vocab_sizecs(fddtjD}|j|S)zReturns vocab as a dictcsi|]}||qSr")convert_ids_to_tokens).0ir;r"r/ sz,GemmaTokenizer.get_vocab..)ranger<r6added_tokens_encoder)r+vocabr"r;r/ get_vocabs zGemmaTokenizer.get_vocabtextr returnc stj|fi|S)ze Args: text: TextInput Simply calls PreTrainedTokenizer's method )r)tokenizer+rEr,r-r"r/rGszGemmaTokenizer.tokenizecKs|jj|tdS)z Args: text: TextInput Returns a tokenized string. The Gemma tokenizer never adds a prefix space. )out_type)r'encoder$rHr"r"r/ _tokenizeszGemmaTokenizer._tokenizecCs |j|S)z0Converts a token (str) in an id using the vocab.)r' piece_to_id)r+tokenr"r"r/_convert_token_to_ids z#GemmaTokenizer._convert_token_to_idcCs|j|}|S)z=Converts an index (integer) in a token (str) using the vocab.)r' IdToPiece)r+indexrMr"r"r/_convert_id_to_tokens z#GemmaTokenizer._convert_id_to_tokencCsTg}d}|D]}||jvr||j||7}g}q||q||j|7}|S)z:Converts a sequence of tokens (string) in a single string.)_added_tokens_encoderr'decodeappend)r+tokenscurrent_sub_tokens out_stringrMr"r"r/convert_tokens_to_strings  z'GemmaTokenizer.convert_tokens_to_stringfilename_prefixcCstj|std|ddStj||r|dndtd}tj|jtj|kr?tj |jr?t |j||fStj |jsgt |d}|j }||Wd|fS1sbwY|fS)a Save the vocabulary and special tokens file to a directory. Args: save_directory (`str`): The directory in which to save the vocabulary. Returns: `Tuple(str)`: Paths to the files saved. zVocabulary path (z) should be a directoryN-rRr wb)ospathisdirloggererrorjoinVOCAB_FILES_NAMESabspathr isfileropenr'r3write)r+save_directoryrZout_vocab_fileficontent_spiece_modelr"r"r/save_vocabularys" (     zGemmaTokenizer.save_vocabularycCsL|jr|jgng}|jr|jgng}|||}|dur$||||}|SN)r bos_token_idr eos_token_idr+ token_ids_0 token_ids_1rnrooutputr"r"r/ build_inputs_with_special_tokenss  z/GemmaTokenizer.build_inputs_with_special_tokensrqrralready_has_special_tokenscs|r tj||ddS|jrdgng}|jrdgng}|dur*|dgt||S|dgt|||dgt||S)a Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. Args: token_ids_0 (`list[int]`): List of IDs. token_ids_1 (`list[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: `list[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. T)rqrrruNr)r)get_special_tokens_maskrrlen)r+rqrrrurnror-r"r/rws(  z&GemmaTokenizer.get_special_tokens_maskcCs`|jr|jgng}|jr|jgng}dgt|||}|dur.|dgt|||7}|S)a Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT sequence pair mask has the following format: ``` 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 | first sequence | second sequence | ``` if token_ids_1 is None, only returns the first portion of the mask (0s). Args: token_ids_0 (`list[int]`): List of ids. token_ids_1 (`list[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: `list[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s). rNrv)rrnrrorxrpr"r"r/$create_token_type_ids_from_sequencess z3GemmaTokenizer.create_token_type_ids_from_sequences token_idsskip_special_tokensr!cKsg}g}|D]+}|r||jvrq||jvr,|r ||j|||j|jg}q||q|r=||j||rEd|}nd|}|tdS)N rR) all_special_ids_added_tokens_decoderrUr'rTcontentrbreplaceSPIECE_UNDERLINE)r+rzr{r!r, sub_textscurrent_sub_textidsr"r"r/_decode1s"     zGemmaTokenizer._decode) rrrrNTFFFFrm)NF)FF) __name__ __module__ __qualname____doc__rcvocab_files_namesmodel_input_namesrdictr$rr*r5r9propertyr<rDlistrGrKrNrQrYtuplerlrtintboolrwryr __classcell__r"r"r-r/r+st.*    &  $r)r]shutilrtypingrrrr r%tokenization_utilsrrutilsr utils.import_utilsr tokenization_utils_baser get_loggerrr`rcrr__all__r"r"r"r/s"      %