{ "backend": "tokenizers", "bos_token": "<|startoftranscript|>", "eos_token": "<|endoftext|>", "model_max_length": 2048, "pad_token": "", "split_special_tokens": false, "tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "", "add_prefix_space": false, "added_tokens_decoder": { "0": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "<|nospeech|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "10": { "content": "<|timestamp|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100": { "content": "<|kn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "101": { "content": "<|kr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "102": { "content": "<|ks|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "103": { "content": "<|kk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "104": { "content": "<|km|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "105": { "content": "<|ki|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "106": { "content": "<|rw|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "107": { "content": "<|ky|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "108": { "content": "<|kv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "109": { "content": "<|kg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "11": { "content": "<|notimestamp|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "110": { "content": "<|ko|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "111": { "content": "<|kj|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "112": { "content": "<|ku|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "113": { "content": "<|lo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "114": { "content": "<|la|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "115": { "content": "<|lv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "116": { "content": "<|li|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "117": { "content": "<|ln|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "118": { "content": "<|lt|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "119": { "content": "<|lu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "12": { "content": "<|diarize|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "120": { "content": "<|lb|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "121": { "content": "<|mk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "122": { "content": "<|mg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "123": { "content": "<|ms|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "124": { "content": "<|ml|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "125": { "content": "<|mt|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "126": { "content": "<|gv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "127": { "content": "<|mi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128": { "content": "<|mr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "129": { "content": "<|mh|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "13": { "content": "<|nodiarize|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130": { "content": "<|mn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131": { "content": "<|na|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "132": { "content": "<|nv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "133": { "content": "<|nd|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "134": { "content": "<|nr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "135": { "content": "<|ng|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "136": { "content": "<|ne|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "137": { "content": "<|no|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "138": { "content": "<|nb|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "139": { "content": "<|nn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "14": { "content": "<|spkchange|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "140": { "content": "<|oc|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "141": { "content": "<|oj|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "142": { "content": "<|or|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "143": { "content": "<|om|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "144": { "content": "<|os|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "145": { "content": "<|pi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "146": { "content": "<|ps|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "147": { "content": "<|fa|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "148": { "content": "<|pl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "149": { "content": "<|pt|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "15": { "content": "<|audioseparator|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "150": { "content": "<|pa|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151": { "content": "<|qu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "152": { "content": "<|ro|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "153": { "content": "<|rm|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "154": { "content": "<|rn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "155": { "content": "<|ru|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "156": { "content": "<|se|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "157": { "content": "<|sm|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "158": { "content": "<|sg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "159": { "content": "<|sa|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16": { "content": "<|emo:undefined|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "160": { "content": "<|sc|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "161": { "content": "<|sr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "162": { "content": "<|sn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "163": { "content": "<|sd|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "164": { "content": "<|si|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "165": { "content": "<|sk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "166": { "content": "<|sl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "167": { "content": "<|so|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "168": { "content": "<|st|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "169": { "content": "<|es|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "17": { "content": "<|emo:neutral|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "170": { "content": "<|su|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "171": { "content": "<|sw|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "172": { "content": "<|ss|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "173": { "content": "<|sv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "174": { "content": "<|tl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "175": { "content": "<|ty|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "176": { "content": "<|tg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "177": { "content": "<|ta|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "178": { "content": "<|tt|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "179": { "content": "<|te|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "18": { "content": "<|emo:happy|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "180": { "content": "<|th|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "181": { "content": "<|bo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "182": { "content": "<|ti|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "183": { "content": "<|to|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "184": { "content": "<|ts|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "185": { "content": "<|tn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "186": { "content": "<|tr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "187": { "content": "<|tk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "188": { "content": "<|tw|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "189": { "content": "<|ug|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "19": { "content": "<|emo:sad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "190": { "content": "<|uk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "191": { "content": "<|ur|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "192": { "content": "<|uz|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "193": { "content": "<|ve|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "194": { "content": "<|vi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "195": { "content": "<|vo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "196": { "content": "<|wa|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "197": { "content": "<|cy|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "198": { "content": "<|wo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "199": { "content": "<|xh|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "20": { "content": "<|emo:angry|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "200": { "content": "<|ii|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "201": { "content": "<|yi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "202": { "content": "<|yo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "203": { "content": "<|za|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "204": { "content": "<|zu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "205": { "content": "<|spk0|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "206": { "content": "<|spk1|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "207": { "content": "<|spk2|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "208": { "content": "<|spk3|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "209": { "content": "<|spk4|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "21": { "content": "<|unklang|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "210": { "content": "<|spk5|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "211": { "content": "<|spk6|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "212": { "content": "<|spk7|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "213": { "content": "<|spk8|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "214": { "content": "<|spk9|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "215": { "content": "<|spk10|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "216": { "content": "<|spk11|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "217": { "content": "<|spk12|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "218": { "content": "<|spk13|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "219": { "content": "<|spk14|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "22": { "content": "<|aa|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "220": { "content": "<|spk15|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "221": { "content": "<|spltoken0|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "222": { "content": "<|spltoken1|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "223": { "content": "<|spltoken2|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "224": { "content": "<|spltoken3|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "225": { "content": "<|spltoken4|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "226": { "content": "<|spltoken5|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "227": { "content": "<|spltoken6|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "228": { "content": "<|spltoken7|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "229": { "content": "<|spltoken8|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "23": { "content": "<|ab|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "230": { "content": "<|spltoken9|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "231": { "content": "<|spltoken10|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "232": { "content": "<|spltoken11|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "233": { "content": "<|spltoken12|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "234": { "content": "<|spltoken13|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "235": { "content": "<|spltoken14|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "236": { "content": "<|spltoken15|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "237": { "content": "<|spltoken16|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "238": { "content": "<|spltoken17|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "239": { "content": "<|spltoken18|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "24": { "content": "<|af|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "240": { "content": "<|spltoken19|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "241": { "content": "<|spltoken20|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "242": { "content": "<|spltoken21|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "243": { "content": "<|spltoken22|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "244": { "content": "<|spltoken23|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "245": { "content": "<|spltoken24|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "246": { "content": "<|spltoken25|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "247": { "content": "<|spltoken26|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "248": { "content": "<|spltoken27|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "249": { "content": "<|spltoken28|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "25": { "content": "<|ak|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "250": { "content": "<|spltoken29|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "251": { "content": "<|spltoken30|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "252": { "content": "<|spltoken31|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "253": { "content": "<|spltoken32|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "254": { "content": "<|spltoken33|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "26": { "content": "<|sq|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "27": { "content": "<|am|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "28": { "content": "<|ar|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "29": { "content": "<|an|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "30": { "content": "<|hy|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "31": { "content": "<|as|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32": { "content": "<|av|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "33": { "content": "<|ae|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "34": { "content": "<|ay|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "35": { "content": "<|az|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "36": { "content": "<|bm|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "37": { "content": "<|ba|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "38": { "content": "<|eu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "39": { "content": "<|be|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "4": { "content": "<|startoftranscript|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "40": { "content": "<|bn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "41": { "content": "<|bi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "42": { "content": "<|bs|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "43": { "content": "<|br|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "44": { "content": "<|bg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "45": { "content": "<|my|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "46": { "content": "<|ca|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "47": { "content": "<|ch|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "48": { "content": "<|ce|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "49": { "content": "<|ny|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "5": { "content": "<|pnc|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50": { "content": "<|zh|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "51": { "content": "<|cu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "52": { "content": "<|cv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "53": { "content": "<|kw|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "54": { "content": "<|co|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "55": { "content": "<|cr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "56": { "content": "<|hr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "57": { "content": "<|cs|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "58": { "content": "<|da|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "59": { "content": "<|dv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "6": { "content": "<|nopnc|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "60": { "content": "<|nl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "61": { "content": "<|dz|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "62": { "content": "<|en|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "63": { "content": "<|eo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "64": { "content": "<|et|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "65": { "content": "<|ee|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "66": { "content": "<|fo|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "67": { "content": "<|fj|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "68": { "content": "<|fi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "69": { "content": "<|fr|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "7": { "content": "<|startofcontext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "70": { "content": "<|fy|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "71": { "content": "<|ff|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "72": { "content": "<|gd|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "73": { "content": "<|gl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "74": { "content": "<|lg|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "75": { "content": "<|ka|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "76": { "content": "<|de|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "77": { "content": "<|el|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "78": { "content": "<|kl|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "79": { "content": "<|gn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "8": { "content": "<|itn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "80": { "content": "<|gu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "81": { "content": "<|ht|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "82": { "content": "<|ha|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "83": { "content": "<|he|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "84": { "content": "<|hz|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "85": { "content": "<|hi|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "86": { "content": "<|ho|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "87": { "content": "<|hu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "88": { "content": "<|is|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "89": { "content": "<|io|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "9": { "content": "<|noitn|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "90": { "content": "<|ig|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "91": { "content": "<|id|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92": { "content": "<|ia|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "93": { "content": "<|ie|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "94": { "content": "<|iu|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "95": { "content": "<|ik|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "96": { "content": "<|ga|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "97": { "content": "<|it|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "98": { "content": "<|ja|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "99": { "content": "<|jv|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "<|nospeech|>", "<|pnc|>", "<|nopnc|>", "<|startofcontext|>", "<|itn|>", "<|noitn|>", "<|timestamp|>", "<|notimestamp|>", "<|diarize|>", "<|nodiarize|>", "<|spkchange|>", "<|audioseparator|>", "<|emo:undefined|>", "<|emo:neutral|>", "<|emo:happy|>", "<|emo:sad|>", "<|emo:angry|>", "<|unklang|>", "<|aa|>", "<|ab|>", "<|af|>", "<|ak|>", "<|sq|>", "<|am|>", "<|ar|>", "<|an|>", "<|hy|>", "<|as|>", "<|av|>", "<|ae|>", "<|ay|>", "<|az|>", "<|bm|>", "<|ba|>", "<|eu|>", "<|be|>", "<|bn|>", "<|bi|>", "<|bs|>", "<|br|>", "<|bg|>", "<|my|>", "<|ca|>", "<|ch|>", "<|ce|>", "<|ny|>", "<|zh|>", "<|cu|>", "<|cv|>", "<|kw|>", "<|co|>", "<|cr|>", "<|hr|>", "<|cs|>", "<|da|>", "<|dv|>", "<|nl|>", "<|dz|>", "<|en|>", "<|eo|>", "<|et|>", "<|ee|>", "<|fo|>", "<|fj|>", "<|fi|>", "<|fr|>", "<|fy|>", "<|ff|>", "<|gd|>", "<|gl|>", "<|lg|>", "<|ka|>", "<|de|>", "<|el|>", "<|kl|>", "<|gn|>", "<|gu|>", "<|ht|>", "<|ha|>", "<|he|>", "<|hz|>", "<|hi|>", "<|ho|>", "<|hu|>", "<|is|>", "<|io|>", "<|ig|>", "<|id|>", "<|ia|>", "<|ie|>", "<|iu|>", "<|ik|>", "<|ga|>", "<|it|>", "<|ja|>", "<|jv|>", "<|kn|>", "<|kr|>", "<|ks|>", "<|kk|>", "<|km|>", "<|ki|>", "<|rw|>", "<|ky|>", "<|kv|>", "<|kg|>", "<|ko|>", "<|kj|>", "<|ku|>", "<|lo|>", "<|la|>", "<|lv|>", "<|li|>", "<|ln|>", "<|lt|>", "<|lu|>", "<|lb|>", "<|mk|>", "<|mg|>", "<|ms|>", "<|ml|>", "<|mt|>", "<|gv|>", "<|mi|>", "<|mr|>", "<|mh|>", "<|mn|>", "<|na|>", "<|nv|>", "<|nd|>", "<|nr|>", "<|ng|>", "<|ne|>", "<|no|>", "<|nb|>", "<|nn|>", "<|oc|>", "<|oj|>", "<|or|>", "<|om|>", "<|os|>", "<|pi|>", "<|ps|>", "<|fa|>", "<|pl|>", "<|pt|>", "<|pa|>", "<|qu|>", "<|ro|>", "<|rm|>", "<|rn|>", "<|ru|>", "<|se|>", "<|sm|>", "<|sg|>", "<|sa|>", "<|sc|>", "<|sr|>", "<|sn|>", "<|sd|>", "<|si|>", "<|sk|>", "<|sl|>", "<|so|>", "<|st|>", "<|es|>", "<|su|>", "<|sw|>", "<|ss|>", "<|sv|>", "<|tl|>", "<|ty|>", "<|tg|>", "<|ta|>", "<|tt|>", "<|te|>", "<|th|>", "<|bo|>", "<|ti|>", "<|to|>", "<|ts|>", "<|tn|>", "<|tr|>", "<|tk|>", "<|tw|>", "<|ug|>", "<|uk|>", "<|ur|>", "<|uz|>", "<|ve|>", "<|vi|>", "<|vo|>", "<|wa|>", "<|cy|>", "<|wo|>", "<|xh|>", "<|ii|>", "<|yi|>", "<|yo|>", "<|za|>", "<|zu|>", "<|spk0|>", "<|spk1|>", "<|spk2|>", "<|spk3|>", "<|spk4|>", "<|spk5|>", "<|spk6|>", "<|spk7|>", "<|spk8|>", "<|spk9|>", "<|spk10|>", "<|spk11|>", "<|spk12|>", "<|spk13|>", "<|spk14|>", "<|spk15|>", "<|spltoken0|>", "<|spltoken1|>", "<|spltoken2|>", "<|spltoken3|>", "<|spltoken4|>", "<|spltoken5|>", "<|spltoken6|>", "<|spltoken7|>", "<|spltoken8|>", "<|spltoken9|>", "<|spltoken10|>", "<|spltoken11|>", "<|spltoken12|>", "<|spltoken13|>", "<|spltoken14|>", "<|spltoken15|>", "<|spltoken16|>", "<|spltoken17|>", "<|spltoken18|>", "<|spltoken19|>", "<|spltoken20|>", "<|spltoken21|>", "<|spltoken22|>", "<|spltoken23|>", "<|spltoken24|>", "<|spltoken25|>", "<|spltoken26|>", "<|spltoken27|>", "<|spltoken28|>", "<|spltoken29|>", "<|spltoken30|>", "<|spltoken31|>", "<|spltoken32|>", "<|spltoken33|>" ], "clean_up_tokenization_spaces": false, "sp_model_kwargs": {} }