o ̳i @sddlmZmZmZmZmZmZddlmZm Z m Z ddl m Z ddl mZddlmZdddd d d d d ddddd ZdZeddD]ZedkrIqBeeded<ed7ZqBdZGddde eZdS))AnyDictListMappingOptionalTuple)MessagePromptTemplatetruncate)ModelTokenizer) Transform)GPT2BaseTokenizer頇iiiiiiiiiii) z <|dummy_0|> <|endoftext|>z<|fim_prefix|>z<|fim_middle|>z<|fim_suffix|>z <|dummy_1|>z <|dummy_2|>z <|dummy_3|> <|im_start|> <|im_end|> <|im_sep|>z<|endofprompt|>iiz<|dummy_z|>zs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+c @seZdZdZ     d#dededeeeefdeedeef dd Z e d d Z d$d ede de de efddZd%de ede defddZdedefddZdddde ede de dee ee e ffddZd eeefdeeeffd!d"ZdS)& Phi4TokenizeraQ TikToken tokenizer configured with Phi4 (14B) special tokens. Args: merges_path (str): Path to merges.txt file. vocab_path (str): Path to vocab.json file. special_tokens (Optional[Dict[str, int]]): Mapping containing special text tokens and their registered token IDs. If left as None, this will be set to the canonical Phi4 special tokens. max_seq_len (Optional[int]): Max sequence length to truncate tokens to. prompt_template (Optional[PromptTemplate]): Template used to format the messages based on their role. N merges_path vocab_pathspecial_tokens max_seq_lenprompt_templatecCsd|pt|_|jd|_|jd|_|jd|_|jg|_||_||_t|||j|j|j|j|_ dS)Nrrz <|dummy_85|>) PHI4_SPECIAL_TOKENSreos_idbos_idpad_id stop_tokensrrr tokenizer_model)selfrrrrrr"T/home/ubuntu/.local/lib/python3.10/site-packages/torchtune/models/phi4/_tokenizer.py__init__6s      zPhi4Tokenizer.__init__cCs|jjS)N)r vocab_size)r!r"r"r#r%RszPhi4Tokenizer.vocab_sizeTtextadd_bosadd_eosreturncCs|jj|||dS)N)r&r'r()r encode)r!r&r'r(r"r"r#r*VszPhi4Tokenizer.encodeidsskip_special_tokenscsfdd|D}|j|S)zDecode token IDs to strings.cs*g|]}rd|krdksn|qS)rir").0token_idr,r"r# ]sz(Phi4Tokenizer.decode..)r decode)r!r+r,ids_for_decoder"r/r#r1[s  zPhi4Tokenizer.decoderolecCs6|jdg}||j|ddd||jd|S)NrFr'r(r)rextendr*append)r!r3tokenized_messagesr"r"r#_tokenize_headerds zPhi4Tokenizer._tokenize_headerF)r(ignore_system_promptmessagesr9c Csl|jr||n|}g}g}|D]}|r|jdkrq||j}||||jgt|g} |jD] } | ddkrM| |j| ddddd7} q5t d| d|rf|jd krf| |j d n |jd krs| |j d || ||jgt| |j rt||j krnq|j rt||j krt ||j |r|jnd}t ||j |r|jnd}||fS) Nsystemtyper&content Fr4z"Unsupported message content type: assistantr)rr3r8r5maskedlenr=r*rstrip RuntimeErrorr6rrr r) r!r:r(r9templated_messagesr7maskmessagetokenized_headertokensitemr"r"r#tokenize_messagesjsD        zPhi4Tokenizer.tokenize_messagessamplecCs,|d}||\}}||d<||d<|S)zR Apply `tokenize_messages` to the "messages" field in the sample. r:rHrE)poprJ)r!rKr:rHrEr"r"r#__call__s zPhi4Tokenizer.__call__)NNNNN)TT)T)__name__ __module__ __qualname____doc__strrrintr r$propertyr%boolrr*r1listr8rrrJrrrMr"r"r"r#r(sX      &5rN)typingrrrrrrtorchtune.datarr r torchtune.modules.tokenizersr torchtune.modules.transformsr 'torchtune.modules.transforms.tokenizersr rcurrent_dummy_indexranger.CL100K_PATTERNrr"r"r"r#s4