o ei@spdZddlmZmZmZmZddlmZddlm Z ddl m Z e e Zddd d ZGd d d e Zd gZd S)z$Tokenization classes for OpenAI GPT.) Tokenizerdecoders normalizerspre_tokenizers)BPE)TokenizersBackend)loggingz vocab.jsonz merges.txtztokenizer.json) vocab_file merges_filetokenizer_filecsneZdZdZeZddgZeZ   d de e e e fBdBde e e BdBde ffd d Z ed d ZZS)OpenAIGPTTokenizera Construct a GPT Tokenizer (backed by HuggingFace's *tokenizers* library). Based on Byte-Pair-Encoding with the following peculiarities: - lower case all inputs - uses BERT's BasicTokenizer for pre-BPE tokenization This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods. Args: vocab_file (`str`, *optional*): Path to the vocabulary file. merges_file (`str`, *optional*): Path to the merges file. tokenizer_file (`str`, *optional*): Path to a tokenizers JSON file containing the serialization of a tokenizer. unk_token (`str`, *optional*, defaults to `""`): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. vocab (`str` or `dict[str, int]`, *optional*): Custom vocabulary dictionary. If not provided, a blank vocabulary is initialized. merges (`str` or `list[str]`, *optional*): Custom merges list. If not provided, an empty list is used. input_idsattention_maskNvocabmerges unk_tokenc s|dur|nt|di|_|pg|_tt|j|jddddt|d|_ttt t g|j_ t |j_tjdd|j_tjdd|i|dS) NrzF)rrdropoutcontinuing_subword_prefixend_of_word_suffixfuse_unkr)suffixr)str_vocab_mergesrr _tokenizerrSequenceNFD Lowercase StripAccents normalizerrBertPreTokenizer pre_tokenizerr BPEDecoderdecodersuper__init__)selfrrrkwargs __class__rl/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/openai/tokenization_openai.pyr);s4    zOpenAIGPTTokenizer.__init__cCsdS)NTr)r*rrr. do_lower_casecsz OpenAIGPTTokenizer.do_lower_case)NNr)__name__ __module__ __qualname____doc__VOCAB_FILES_NAMESvocab_files_namesmodel_input_namesrmodelrdictintlistr)propertyr/ __classcell__rrr,r.r s"(r N)r3 tokenizersrrrrtokenizers.modelsrtokenization_utils_tokenizersrutilsr get_loggerr0loggerr4r __all__rrrr.s       L