# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/1b.indic_normalizer.ipynb. # %% auto 0 __all__ = ['NormalizerI', 'BaseNormalizer', 'DevanagariNormalizer', 'HindiNormalizer', 'PunjabiNormalizer', 'TeluguNormalizer', 'GujaratiNormalizer', 'OdiaNormalizer', 'BengaliNormalizer', 'TamilNormalizer', 'KannadaNormalizer', 'MalayalamNormalizer'] # %% ../nbs/1b.indic_normalizer.ipynb 3 import re from indic_numtowords import num2words from . import langinfo # %% ../nbs/1b.indic_normalizer.ipynb 4 # Copyright (c) 2013-present, Anoop Kunchukuttan # All rights reserved. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. # # Program for normalization of text written in Unicode. This is mainly geared towards Indic scripts # # @author Anoop Kunchukuttan # class NormalizerI(object): """ The normalizer classes do the following: * Some characters have multiple Unicode codepoints. The normalizer chooses a single standard representation * Some control characters are deleted * While typing using the Latin keyboard, certain typical mistakes occur which are corrected by the module Base class for normalizer. Performs some common normalization, which includes: * Byte order mark, word joiner, etc. removal * ZERO_WIDTH_NON_JOINER and ZERO_WIDTH_JOINER removal * ZERO_WIDTH_SPACE and NO_BREAK_SPACE replaced by spaces Script specific normalizers should derive from this class and override the normalize() method. They can call the super class 'normalize() method to avail of the common normalization """ BYTE_ORDER_MARK = "\ufeff" BYTE_ORDER_MARK_2 = "\ufffe" WORD_JOINER = "\u2060" SOFT_HYPHEN = "\u00ad" ZERO_WIDTH_SPACE = "\u200b" NO_BREAK_SPACE = "\u00a0" ZERO_WIDTH_NON_JOINER = "\u200c" ZERO_WIDTH_JOINER = "\u200d" def _normalize_punctuations(self, text): """ Normalize punctuations. Applied many of the punctuation normalizations that are part of MosesNormalizer from sacremoses """ text = text.replace(NormalizerI.BYTE_ORDER_MARK, "") text = text.replace("„", r'"') text = text.replace("“", r'"') text = text.replace("”", r'"') text = text.replace("–", r"-") text = text.replace("—", r" - ") text = text.replace("´", r"'") text = text.replace("‘", r"'") text = text.replace("‚", r"'") text = text.replace("’", r"'") text = text.replace("''", r'"') text = text.replace("´´", r'"') text = text.replace("…", r"...") return text def normalize(self, text): pass # %% ../nbs/1b.indic_normalizer.ipynb 5 class BaseNormalizer(NormalizerI): """Common class used in most of indic languages inherit from this code.""" def __init__( self, lang, remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, ): self.lang = lang self.remove_nuktas = remove_nuktas self.nasals_mode = nasals_mode self.do_normalize_chandras = do_normalize_chandras self.do_normalize_vowel_ending = do_normalize_vowel_ending self._init_normalize_chandras() self._init_normalize_nasals() self._init_normalize_vowel_ending() # self._init_visarga_correction() def _init_normalize_vowel_ending(self): if self.lang in langinfo.IE_LANGUAGES: self.fn_vowel_ending = self._normalize_word_vowel_ending_ie elif self.lang in langinfo.DRAVIDIAN_LANGUAGES: self.fn_vowel_ending = self._normalize_word_vowel_ending_dravidian else: self.fn_vowel_ending = lambda x: x def _init_normalize_chandras(self): substitution_offsets = [ [0x0D, 0x0F], # chandra e, independent [0x11, 0x13], # chandra o, independent [0x45, 0x47], # chandra e , 0xde],pendent [0x49, 0x4B], # chandra o , 0xde],pendent # [0x72 , 0x0f], # mr: chandra e, independent [0x00, 0x02], # chandrabindu [0x01, 0x02], # chandrabindu ] self.chandra_substitutions = [ ( langinfo.offset_to_char(x[0], self.lang), langinfo.offset_to_char(x[1], self.lang), ) for x in substitution_offsets ] def _normalize_chandras(self, text): for match, repl in self.chandra_substitutions: text = text.replace(match, repl) return text def _init_to_anusvaara_strict(self): """ `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')` """ pat_signatures = [ [0x19, 0x15, 0x18], [0x1E, 0x1A, 0x1D], [0x23, 0x1F, 0x22], [0x28, 0x24, 0x27], [0x29, 0x24, 0x27], [0x2E, 0x2A, 0x2D], ] halant_offset = 0x4D anusvaara_offset = 0x02 pats = [] for pat_signature in pat_signatures: pat = re.compile( r"{nasal}{halant}([{start_r}-{end_r}])".format( nasal=langinfo.offset_to_char(pat_signature[0], self.lang), halant=langinfo.offset_to_char(halant_offset, self.lang), start_r=langinfo.offset_to_char(pat_signature[1], self.lang), end_r=langinfo.offset_to_char(pat_signature[2], self.lang), ) ) pats.append(pat) repl_string = "{anusvaara}\\1".format( anusvaara=langinfo.offset_to_char(anusvaara_offset, self.lang) ) self.pats_repls = (pats, repl_string) def _to_anusvaara_strict(self, text): pats, repl_string = self.pats_repls for pat in pats: text = pat.sub(repl_string, text) return text def _init_to_anusvaara_relaxed(self): """ `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')` """ nasals_list = [0x19, 0x1E, 0x23, 0x28, 0x29, 0x2E] nasals_list_str = ",".join( [langinfo.offset_to_char(x, self.lang) for x in nasals_list] ) halant_offset = 0x4D anusvaara_offset = 0x02 pat = re.compile( r"[{nasals_list_str}]{halant}".format( nasals_list_str=nasals_list_str, halant=langinfo.offset_to_char(halant_offset, self.lang), ) ) repl_string = "{anusvaara}".format( anusvaara=langinfo.offset_to_char(anusvaara_offset, self.lang) ) self.pats_repls = (pat, repl_string) def _to_anusvaara_relaxed(self, text): pat, repl_string = self.pats_repls return pat.sub(repl_string, text) def _init_to_nasal_consonants(self): """ `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')` """ pat_signatures = [ [0x19, 0x15, 0x18], [0x1E, 0x1A, 0x1D], [0x23, 0x1F, 0x22], [0x28, 0x24, 0x27], [0x29, 0x24, 0x27], [0x2E, 0x2A, 0x2D], ] halant_offset = 0x4D anusvaara_offset = 0x02 pats = [] repl_strings = [] for pat_signature in pat_signatures: pat = re.compile( r"{anusvaara}([{start_r}-{end_r}])".format( anusvaara=langinfo.offset_to_char(anusvaara_offset, self.lang), start_r=langinfo.offset_to_char(pat_signature[1], self.lang), end_r=langinfo.offset_to_char(pat_signature[2], self.lang), ) ) pats.append(pat) repl_string = "{nasal}{halant}\\1".format( nasal=langinfo.offset_to_char(pat_signature[0], self.lang), halant=langinfo.offset_to_char(halant_offset, self.lang), ) repl_strings.append(repl_string) self.pats_repls = list(zip(pats, repl_strings)) def _to_nasal_consonants(self, text): for pat, repl in self.pats_repls: text = pat.sub(repl, text) return text def _init_normalize_nasals(self): if self.nasals_mode == "to_anusvaara_strict": self._init_to_anusvaara_strict() elif self.nasals_mode == "to_anusvaara_relaxed": self._init_to_anusvaara_relaxed() elif self.nasals_mode == "to_nasal_consonants": self._init_to_nasal_consonants() def _normalize_nasals(self, text): if self.nasals_mode == "to_anusvaara_strict": return self._to_anusvaara_strict(text) elif self.nasals_mode == "to_anusvaara_relaxed": return self._to_anusvaara_relaxed(text) elif self.nasals_mode == "to_nasal_consonants": return self._to_nasal_consonants(text) else: return text def _normalize_word_vowel_ending_dravidian(self, word): """ for Dravidian - consonant ending: add 'a' ki maatra - halant ending: no change - 'a' ki maatra: no change """ if len(word) > 0 and langinfo.is_consonant(word[-1], self.lang): return word + langinfo.offset_to_char(0x3E, self.lang) else: return word def _normalize_word_vowel_ending_ie(self, word): """ for IE - consonant ending: add halant - halant ending: no change - 'a' ki maatra: no change """ if len(word) > 0 and langinfo.is_consonant(word[-1], self.lang): return word + langinfo.offset_to_char(langinfo.HALANTA_OFFSET, self.lang) else: return word def _normalize_vowel_ending(self, text): return " ".join([self.fn_vowel_ending(w) for w in text.split(" ")]) def normalize(self, text): """ Method to be implemented for normalization for each script """ text = text.replace(NormalizerI.BYTE_ORDER_MARK, "") text = text.replace(NormalizerI.BYTE_ORDER_MARK_2, "") text = text.replace(NormalizerI.WORD_JOINER, "") text = text.replace(NormalizerI.SOFT_HYPHEN, "") text = text.replace(NormalizerI.ZERO_WIDTH_SPACE, " ") # ?? text = text.replace(NormalizerI.NO_BREAK_SPACE, " ") text = text.replace(NormalizerI.ZERO_WIDTH_NON_JOINER, "") text = text.replace(NormalizerI.ZERO_WIDTH_JOINER, "") text = self._normalize_punctuations(text) if self.do_normalize_chandras: text = self._normalize_chandras(text) text = self._normalize_nasals(text) if self.do_normalize_vowel_ending: text = self._normalize_vowel_ending(text) return text def get_char_stats(self, text): print(len(re.findall(NormalizerI.BYTE_ORDER_MARK, text))) print(len(re.findall(NormalizerI.BYTE_ORDER_MARK_2, text))) print(len(re.findall(NormalizerI.WORD_JOINER, text))) print(len(re.findall(NormalizerI.SOFT_HYPHEN, text))) print(len(re.findall(NormalizerI.ZERO_WIDTH_SPACE, text))) print(len(re.findall(NormalizerI.NO_BREAK_SPACE, text))) print(len(re.findall(NormalizerI.ZERO_WIDTH_NON_JOINER, text))) print(len(re.findall(NormalizerI.ZERO_WIDTH_JOINER, text))) # for mobj in re.finditer(NormalizerI.ZERO_WIDTH_NON_JOINER,text): # print text[mobj.start()-10:mobj.end()+10].replace('\n', ' ').replace(NormalizerI.ZERO_WIDTH_NON_JOINER,'').encode('utf-8') # print hex(ord(text[mobj.end():mobj.end()+1])) def correct_visarga(self, text, visarga_char, char_range): text = re.sub(r"([\u0900-\u097f]):", "\\1\u0903", text) # %% ../nbs/1b.indic_normalizer.ipynb 6 class DevanagariNormalizer(BaseNormalizer): """ Normalizer for the Devanagari script. In addition to basic normalization by the super class, * Replaces the composite characters containing nuktas by their decomposed form * replace pipe character '|' by poorna virama character * replace colon ':' by visarga if the colon follows a charcter in this script """ NUKTA = "\u093c" def __init__( self, lang="hi", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, ): super(DevanagariNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) def __call__(self, text): # common normalization for Indic scripts text = super(DevanagariNormalizer, self).normalize(text) # chandra a replacement for Marathi text = text.replace("\u0972", "\u090f") # decomposing Nukta based composite characters text = text.replace("\u0929", "\u0928" + DevanagariNormalizer.NUKTA) text = text.replace("\u0931", "\u0930" + DevanagariNormalizer.NUKTA) text = text.replace("\u0934", "\u0933" + DevanagariNormalizer.NUKTA) text = text.replace("\u0958", "\u0915" + DevanagariNormalizer.NUKTA) text = text.replace("\u0959", "\u0916" + DevanagariNormalizer.NUKTA) text = text.replace("\u095a", "\u0917" + DevanagariNormalizer.NUKTA) text = text.replace("\u095b", "\u091c" + DevanagariNormalizer.NUKTA) text = text.replace("\u095c", "\u0921" + DevanagariNormalizer.NUKTA) text = text.replace("\u095d", "\u0922" + DevanagariNormalizer.NUKTA) text = text.replace("\u095e", "\u092b" + DevanagariNormalizer.NUKTA) text = text.replace("\u095f", "\u092f" + DevanagariNormalizer.NUKTA) if self.remove_nuktas: text = text.replace(DevanagariNormalizer.NUKTA, "") # replace pipe character for poorna virama text = text.replace("\u007c", "\u0964") # correct visarga text = re.sub(r"([\u0900-\u097f]):", "\\1\u0903", text) has_digits = any(char.isdigit() for char in text) if has_digits: # Extract parts containing digits digit_parts = re.findall(r"\d+", text) for part in digit_parts: text = text.replace(part, num2words(part, lang="hi")) return text def get_char_stats(self, text): super(DevanagariNormalizer, self).get_char_stats(text) print((len(re.findall("\u0929", text)))) print((len(re.findall("\u0931", text)))) print((len(re.findall("\u0934", text)))) print((len(re.findall("\u0958", text)))) print((len(re.findall("\u0959", text)))) print((len(re.findall("\u095a", text)))) print((len(re.findall("\u095b", text)))) print((len(re.findall("\u095c", text)))) print((len(re.findall("\u095d", text)))) print((len(re.findall("\u095e", text)))) print((len(re.findall("\u095f", text)))) # %% ../nbs/1b.indic_normalizer.ipynb 10 class HindiNormalizer(BaseNormalizer): """Fork of Devanagiri normalizer. With additional changes for Hindi and tts_mode.""" NUKTA = "\u093c" def __init__( self, lang="hi", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, tts_mode=False, ): super(HindiNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) self.tts_mode = tts_mode def __call__(self, text): # common normalization for Indic scripts text = super(HindiNormalizer, self).normalize(text) # chandra a replacement for Marathi text = text.replace("\u0972", "\u090f") # decomposing Nukta based composite characters text = text.replace("\u0929", "\u0928" + HindiNormalizer.NUKTA) text = text.replace("\u0931", "\u0930" + HindiNormalizer.NUKTA) text = text.replace("\u0934", "\u0933" + HindiNormalizer.NUKTA) text = text.replace("\u0958", "\u0915" + HindiNormalizer.NUKTA) text = text.replace("\u0959", "\u0916" + HindiNormalizer.NUKTA) text = text.replace("\u095a", "\u0917" + HindiNormalizer.NUKTA) text = text.replace("\u095b", "\u091c" + HindiNormalizer.NUKTA) text = text.replace("\u095c", "\u0921" + HindiNormalizer.NUKTA) text = text.replace("\u095d", "\u0922" + HindiNormalizer.NUKTA) text = text.replace("\u095e", "\u092b" + HindiNormalizer.NUKTA) text = text.replace("\u095f", "\u092f" + HindiNormalizer.NUKTA) if self.remove_nuktas: text = text.replace(HindiNormalizer.NUKTA, "") # replace pipe character for poorna virama text = text.replace("\u007c", "\u0964") # correct visarga text = re.sub(r"([\u0900-\u097f]):", "\\1\u0903", text) if self.tts_mode: # Handle currencies text = re.sub(r"INR\s+(\d+)", r"रुपये \1", text) text = re.sub(r"Rs\.\s+(\d+)", r"रुपये \1", text) text = re.sub(r"₹\s*(\d+)", r"रुपये \1", text) text = re.sub(r"USD\s+(\d+)", r"डॉलर \1", text) text = re.sub(r"\$\s*(\d+)", r"डॉलर \1", text) text = re.sub(r"KRW\s+(\d+)", r"कोरियाई वॉन \1", text) text = re.sub(r"₩\s*(\d+)", r"कोरियाई वॉन \1", text) # Handle decimal numbers def replace_decimal(match): whole, frac = match.group(1), match.group(2) # Convert both parts to Hindi words whole_words = num2words(whole, lang="hi") frac_words = " ".join(num2words(digit, lang="hi") for digit in frac) return f"{whole_words} पॉइंट {frac_words}" text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text) # Normalize full URLs def normalize_url(match): url = match.group(0) scheme = "" if url.startswith("https://"): scheme = "एच टी टी पी एस कोलन स्लैश स्लैश " url = url[len("https://") :] elif url.startswith("http://"): scheme = "एच टी टी पी कोलन स्लैश स्लैश " url = url[len("http://") :] parts = url.split("/", 1) domain = parts[0] path = parts[1] if len(parts) > 1 else "" domain_parts = domain.split(".") spoken_domain = " डॉट ".join( " ".join(part.upper()) for part in domain_parts ) spoken_path = "" if path: path_parts = path.split("/") spoken_path = " स्लैश ".join( " ".join(p.upper()) for p in path_parts if p ) spoken_path = " स्लैश " + spoken_path if spoken_path else "" return f"{scheme}{spoken_domain}{spoken_path}".strip() text = re.sub(r"https?://[^\s]+", normalize_url, text) # Normalize bare domains like www.amazon.in, openai.com def normalize_bare_url(match): domain_parts = match.group(0).split(".") return " डॉट ".join(" ".join(part.upper()) for part in domain_parts) bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b" text = re.sub( bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE ) # Email normalization text = re.sub( r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b", lambda m: f"{m.group(1)} एट {m.group(2)} डॉट {m.group(3)}", text, ) # Replace standalone special characters symbol_map = { "&": " or ", "@": " at ", "%": " percent ", "+": " plus ", "-": " minus ", "=": " equals ", "*": " star ", "#": " hash ", } for sym, word in symbol_map.items(): text = text.replace(sym, word) def expand_acronyms(match): return " ".join(match.group(0)) text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text) text = text.lower() has_digits = any(char.isdigit() for char in text) if has_digits: # digit_parts = re.findall(r"\d+", text) digit_parts = re.findall(r"\d+\.\d+|\d+", text) for x in digit_parts: text = text.replace(x, num2words(x, lang="hi")) return text # %% ../nbs/1b.indic_normalizer.ipynb 17 class PunjabiNormalizer(BaseNormalizer): """ Normalizer for the Gurmukhi script. In addition to basic normalization by the super class, * Replaces the composite characters containing nuktas by their decomposed form * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama * replace pipe character '|' by poorna virama character * replace colon ':' by visarga if the colon follows a charcter in this script """ NUKTA = "\u0a3c" VOWEL_NORM_MAPS = { ## http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf ## Table 12-16 "\u0a05\u0a3e": "\u0a06", "\u0a72\u0a3f": "\u0a07", "\u0a72\u0a40": "\u0a08", "\u0a73\u0a41": "\u0a09", "\u0a73\u0a42": "\u0a0a", "\u0a72\u0a47": "\u0a0f", "\u0a05\u0a48": "\u0a10", "\u0a73\u0a4b": "\u0a13", "\u0a05\u0a4c": "\u0a14", } def __init__( self, lang="pa", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, do_canonicalize_addak=False, do_canonicalize_tippi=False, do_replace_vowel_bases=False, tts_mode=False, ): super(PunjabiNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) self.do_canonicalize_addak = do_canonicalize_addak self.do_canonicalize_tippi = do_canonicalize_tippi self.do_replace_vowel_bases = do_replace_vowel_bases self.tts_mode = tts_mode def _normalize_vowels(self, text): """ """ ## standard vowel replacements as per suggestions in ## http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf ## Table 12-16 for k, v in PunjabiNormalizer.VOWEL_NORM_MAPS.items(): text = text.replace(k, v) ## the above mappings should account for majority of the variantions, ## Rest are handled via this generic rule which looks at the diacritic ## following the 2 special characters ## TBD: don't see evidence for this in Wikipedia corpus ## If these special characters occur without any diacritic, replace them with closet ## equivalent vowels if self.do_replace_vowel_bases: text = text.replace("\u0a72", "\u0a07") text = text.replace("\u0a73", "\u0a09") return text def __call__(self, text): # Addak if self.do_canonicalize_addak: ## replace addak+consonant with consonat+halant+consonant text = re.sub(r"\u0a71(.)", "\1\u0a4d\1", text) # Tippi if self.do_canonicalize_tippi: text = text.replace("\u0a70", "\u0a02") # Vowels: Gurumuki has multiple ways of representing independent vowels due # to the characters 'iri' and 'ura'. text = self._normalize_vowels(text) # common normalization for Indic scripts text = super(PunjabiNormalizer, self).normalize(text) # decomposing Nukta based composite characters text = text.replace("\u0a33", "\u0a32" + PunjabiNormalizer.NUKTA) text = text.replace("\u0a36", "\u0a38" + PunjabiNormalizer.NUKTA) text = text.replace("\u0a59", "\u0a16" + PunjabiNormalizer.NUKTA) text = text.replace("\u0a5a", "\u0a17" + PunjabiNormalizer.NUKTA) text = text.replace("\u0a5b", "\u0a1c" + PunjabiNormalizer.NUKTA) text = text.replace("\u0a5e", "\u0a2b" + PunjabiNormalizer.NUKTA) if self.remove_nuktas: text = text.replace(PunjabiNormalizer.NUKTA, "") # replace the poorna virama codes specific to script # with generic Indic script codes text = text.replace("\u0a64", "\u0964") text = text.replace("\u0a65", "\u0965") ## replace pipe character for poorna virama text = text.replace("\u007c", "\u0964") # correct visarge text = re.sub(r"([\u0a00-\u0a7f]):", "\1\u0a03", text) if self.tts_mode: # Handle currencies text = re.sub(r"INR\s+(\d+)", r"ਰੁਪਏ \1", text) text = re.sub(r"Rs\.\s+(\d+)", r"ਰੁਪਏ \1", text) text = re.sub(r"₹\s*(\d+)", r"ਰੁਪਏ \1", text) text = re.sub(r"USD\s+(\d+)", r"ਡਾਲਰ \1", text) text = re.sub(r"$\s*(\d+)", r"ਡਾਲਰ \1", text) text = re.sub(r"KRW\s+(\d+)", r"ਕੋਰੀਆਈ ਵੌਨ \1", text) text = re.sub(r"₩\s*(\d+)", r"ਕੋਰੀਆਈ ਵੌਨ \1", text) # Handle decimal numbers def replace_decimal(match): whole, frac = match.group(1), match.group(2) return f"{whole} ਪੌਇੰਟ {frac}" text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text) # Normalize full URLs def normalize_url(match): url = match.group(0) scheme = "" if url.startswith("https://"): scheme = "ਐਚ ਟੀ ਟੀ ਪੀ ਐੱਸ ਕੋਲਨ ਸਲੈਸ਼ ਸਲੈਸ਼ " url = url[len("https://") :] elif url.startswith("http://"): scheme = "ਐਚ ਟੀ ਟੀ ਪੀ ਕੋਲਨ ਸਲੈਸ਼ ਸਲੈਸ਼ " url = url[len("http://") :] parts = url.split("/", 1) domain = parts[0] path = parts[1] if len(parts) > 1 else "" domain_parts = domain.split(".") spoken_domain = " ਡਾਟ ".join( " ".join(part.upper()) for part in domain_parts ) spoken_path = "" if path: path_parts = path.split("/") spoken_path = " ਸਲੈਸ਼ ".join( " ".join(p.upper()) for p in path_parts if p ) spoken_path = " ਸਲੈਸ਼ " + spoken_path if spoken_path else "" return f"{scheme}{spoken_domain}{spoken_path}".strip() text = re.sub(r"https?://[^\s]+", normalize_url, text) # Normalize bare domains (e.g., www.amazon.in) def normalize_bare_url(match): domain_parts = match.group(0).split(".") return " ਡਾਟ ".join(" ".join(part.upper()) for part in domain_parts) bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b" text = re.sub( bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE ) # Email normalization text = re.sub( r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b", lambda m: f"{m.group(1)} ਐਟ {m.group(2)} ਡਾਟ {m.group(3)}", text, ) # Replace standalone special characters symbol_map = { "&": " or ", "@": " at ", "%": " percent ", "+": " plus ", "-": " minus ", "=": " equals ", "*": " star ", "#": " hash ", } for sym, word in symbol_map.items(): text = text.replace(sym, word) def expand_acronyms(match): return " ".join(match.group(0)) text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text) has_digits = any(char.isdigit() for char in text) if has_digits: # digit_parts = re.findall(r"\d+", text) digit_parts = re.findall(r"\d+\.\d+|\d+", text) for x in digit_parts: text = text.replace(x, num2words(x, lang="pa")) return text # %% ../nbs/1b.indic_normalizer.ipynb 20 class TeluguNormalizer(BaseNormalizer): """ Normalizer for the Teluguscript. In addition to basic normalization by the super class, * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama * canonicalize two-part dependent vowel signs * replace colon ':' by visarga if the colon follows a charcter in this script """ def __init__( self, lang="te", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, tts_mode=False, ): super(TeluguNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) self.tts_mode = tts_mode def __call__(self, text: str): # common normalization for Indic scripts text = super(TeluguNormalizer, self).normalize(text) # replace the poorna virama codes specific to script # with generic Indic script codes text = text.replace("\u0c64", "\u0964") text = text.replace("\u0c65", "\u0965") # dependent vowels text = text.replace("\u0c46\u0c56", "\u0c48") # correct visarge text = re.sub(r"([\u0c00-\u0c7f]):", "\1\u0c03", text) if self.tts_mode: # Handle currencies text = re.sub(r"INR\s+(\d+)", r"రూపాయలు \1", text) text = re.sub(r"Rs\.\s+(\d+)", r"రూపాయలు \1", text) text = re.sub(r"₹\s*(\d+)", r"రూపాయలు \1", text) text = re.sub(r"USD\s+(\d+)", r"డాలర్లు \1", text) text = re.sub(r"$\s*(\d+)", r"డాలర్లు \1", text) text = re.sub(r"KRW\s+(\d+)", r"కొరియన్ వాన్ \1", text) text = re.sub(r"₩\s*(\d+)", r"కొరియన్ వాన్ \1", text) # Handle decimal numbers def replace_decimal(match): whole, frac = match.group(1), match.group(2) return f"{whole} పాయింట్ {frac}" text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text) # Normalize full URLs def normalize_url(match): url = match.group(0) scheme = "" if url.startswith("https://"): scheme = "హెచ్ టి టి పి ఎస్ కొలన్ స్లాష్ స్లాష్ " url = url[len("https://") :] elif url.startswith("http://"): scheme = "హెచ్ టి టి పి కొలన్ స్లాష్ స్లాష్ " url = url[len("http://") :] parts = url.split("/", 1) domain = parts[0] path = parts[1] if len(parts) > 1 else "" domain_parts = domain.split(".") spoken_domain = " డాట్ ".join( " ".join(part.upper()) for part in domain_parts ) spoken_path = "" if path: path_parts = path.split("/") spoken_path = " స్లాష్ ".join( " ".join(p.upper()) for p in path_parts if p ) spoken_path = " స్లాష్ " + spoken_path if spoken_path else "" return f"{scheme}{spoken_domain}{spoken_path}".strip() text = re.sub(r"https?://[^\s]+", normalize_url, text) # Normalize bare domain names def normalize_bare_url(match): domain_parts = match.group(0).split(".") return " డాట్ ".join(" ".join(part.upper()) for part in domain_parts) bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b" text = re.sub( bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE ) # Email normalization text = re.sub( r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b", lambda m: f"{m.group(1)} అట్ {m.group(2)} డాట్ {m.group(3)}", text, ) # Replace standalone special characters symbol_map = { "&": " or ", "@": " at ", "%": " percent ", "+": " plus ", "-": " minus ", "=": " equals ", "*": " star ", "#": " hash ", } for sym, word in symbol_map.items(): text = text.replace(sym, word) def expand_acronyms(match): return " ".join(match.group(0)) text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text) has_digits = any(char.isdigit() for char in text) if has_digits: digit_parts = re.findall(r"\d+", text) for part in digit_parts: text = text.replace(part, num2words(part, lang="te")) return text.lower() # %% ../nbs/1b.indic_normalizer.ipynb 23 class GujaratiNormalizer(BaseNormalizer): """ Normalizer for the Gujarati script. In addition to basic normalization by the super class, * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama * replace colon ':' by visarga if the colon follows a charcter in this script """ NUKTA = "\u0abc" def __init__( self, lang="gu", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, tts_mode=False, ): super(GujaratiNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) self.tts_mode = tts_mode def __call__(self, text: str): # common normalization for Indic scripts text = super(GujaratiNormalizer, self).normalize(text) # decomposing Nukta based composite characters if self.remove_nuktas: text = text.replace(GujaratiNormalizer.NUKTA, "") # replace the poorna virama codes specific to script # with generic Indic script codes text = text.replace("\u0ae4", "\u0964") text = text.replace("\u0ae5", "\u0965") # correct visarge text = re.sub(r"([\u0a80-\u0aff]):", "\\1\u0a83", text) if self.tts_mode: # Handle currencies text = re.sub(r"INR\s+(\d+)", r"રૂપિયા \1", text) text = re.sub(r"Rs\.\s+(\d+)", r"રૂપિયા \1", text) text = re.sub(r"₹\s*(\d+)", r"રૂપિયા \1", text) text = re.sub(r"USD\s+(\d+)", r"ડોલર \1", text) text = re.sub(r"\$\s*(\d+)", r"ડોલર \1", text) text = re.sub(r"KRW\s+(\d+)", r"કોરિયન વોન \1", text) text = re.sub(r"₩\s*(\d+)", r"કોરિયન વોન \1", text) # Handle decimal numbers def replace_decimal(match): whole, frac = match.group(1), match.group(2) return f"{whole} પોઈન્ટ {frac}" text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text) # Normalize full URLs def normalize_url(match): url = match.group(0) scheme = "" if url.startswith("https://"): scheme = "એચ ટી ટી પીએસ કોલન સ્લેશ સ્લેશ " url = url[len("https://") :] elif url.startswith("http://"): scheme = "એચ ટી ટી પી કોલન સ્લેશ સ્લેશ " url = url[len("http://") :] parts = url.split("/", 1) domain = parts[0] path = parts[1] if len(parts) > 1 else "" domain_parts = domain.split(".") spoken_domain = " ડોટ ".join( " ".join(part.upper()) for part in domain_parts ) spoken_path = "" if path: path_parts = path.split("/") spoken_path = " સ્લેશ ".join( " ".join(p.upper()) for p in path_parts if p ) spoken_path = " સ્લેશ " + spoken_path if spoken_path else "" return f"{scheme}{spoken_domain}{spoken_path}".strip() text = re.sub(r"https?://[^\s]+", normalize_url, text) # Normalize bare domain names def normalize_bare_url(match): domain_parts = match.group(0).split(".") return " ડોટ ".join(" ".join(part.upper()) for part in domain_parts) bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b" text = re.sub( bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE ) # Normalize email addresses text = re.sub( r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b", lambda m: f"{m.group(1)} એટ {m.group(2)} ડોટ {m.group(3)}", text, ) symbol_map = { "&": " or ", "@": " at ", "%": " percent ", "+": " plus ", "-": " minus ", "=": " equals ", "*": " star ", "#": " hash ", } for sym, word in symbol_map.items(): text = text.replace(sym, word) def expand_acronyms(match): return " ".join(match.group(0)) text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text) has_digits = any(char.isdigit() for char in text) if has_digits: digit_parts = re.findall(r"\d+", text) for part in digit_parts: text = text.replace(part, num2words(part, lang="gu")) return text.lower() # %% ../nbs/1b.indic_normalizer.ipynb 25 class OdiaNormalizer(BaseNormalizer): """ Normalizer for the Oriya script. In addition to basic normalization by the super class, * Replaces the composite characters containing nuktas by their decomposed form * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama * Canonicalize two part dependent vowels * Replace 'va' with 'ba' * replace pipe character '|' by poorna virama character * replace colon ':' by visarga if the colon follows a charcter in this script """ NUKTA = "\u0b3c" VOWEL_NORM_MAPS = { ## See Table 12-22 in http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf "\u0b05\u0b3e": "\u0b06", "\u0b0f\u0b57": "\u0b10", "\u0b13\u0b57": "\u0b14", } def __init__( self, lang="or", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, do_remap_wa=False, tts_mode=False, ): super(OdiaNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) self.do_remap_wa = do_remap_wa self.tts_mode = tts_mode def __call__(self, text: str): # common normalization for Indic scripts text = super(OdiaNormalizer, self).normalize(text) ## standard vowel replacements as per suggestions in Unicode documents for k, v in OdiaNormalizer.VOWEL_NORM_MAPS.items(): text = text.replace(k, v) # decomposing Nukta based composite characters text = text.replace("\u0b5c", "\u0b21" + OdiaNormalizer.NUKTA) text = text.replace("\u0b5d", "\u0b22" + OdiaNormalizer.NUKTA) if self.remove_nuktas: text = text.replace(OdiaNormalizer.NUKTA, "") # replace the poorna virama codes specific to script # with generic Indic script codes text = text.replace("\u0b64", "\u0964") text = text.replace("\u0b65", "\u0965") # replace pipe character for poorna virama text = text.replace("\u0b7c", "\u0964") # replace wa with ba if self.do_remap_wa: text = text.replace("\u0b71", "\u0b2c") # replace va with ba # NOTE: documentation (chapter on Indic scripts) and codepoint chart seem contradictory # (this applied to wa to ba rule also above) text = text.replace("\u0b35", "\u0b2c") # AI dependent vowel sign text = text.replace("\u0b47\u0b56", "\u0b58") # two part dependent vowels text = text.replace("\u0b47\u0b3e", "\u0b4b") text = text.replace("\u0b47\u0b57", "\u0b4c") # additional consonant - not clear how to handle this # ignore # correct visarge text = re.sub(r"([\u0b00-\u0b7f]):", "\\1\u0b03", text) if self.tts_mode: # Handle currencies text = re.sub(r"INR\s+(\d+)", r"ଟଙ୍କା \1", text) text = re.sub(r"Rs\.\s+(\d+)", r"ଟଙ୍କା \1", text) text = re.sub(r"₹\s*(\d+)", r"ଟଙ୍କା \1", text) text = re.sub(r"USD\s+(\d+)", r"ଡଲାର \1", text) text = re.sub(r"\$\s*(\d+)", r"ଡଲାର \1", text) text = re.sub(r"KRW\s+(\d+)", r"କୋରିଆନ୍ ୱନ୍ \1", text) text = re.sub(r"₩\s*(\d+)", r"କୋରିଆନ୍ ୱନ୍ \1", text) # Handle decimal numbers def replace_decimal(match): whole, frac = match.group(1), match.group(2) return f"{whole} ପଏଣ୍ଟ {frac}" text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text) def normalize_url(match): url = match.group(0) scheme = "" if url.startswith("https://"): scheme = "ଏଚ୍ ଟି ଟି ପି ଏସ୍ କୋଲନ୍ ସ୍ଲାଶ୍ ସ୍ଲାଶ୍ " url = url[len("https://") :] elif url.startswith("http://"): scheme = "ଏଚ୍ ଟି ଟି ପି କୋଲନ୍ ସ୍ଲାଶ୍ ସ୍ଲାଶ୍ " url = url[len("http://") :] parts = url.split("/", 1) domain = parts[0] path = parts[1] if len(parts) > 1 else "" domain_parts = domain.split(".") spoken_domain = " ଡଟ୍ ".join( " ".join(part.upper()) for part in domain_parts ) spoken_path = "" if path: path_parts = path.split("/") spoken_path = " ସ୍ଲାଶ୍ ".join( " ".join(p.upper()) for p in path_parts if p ) spoken_path = " ସ୍ଲାଶ୍ " + spoken_path if spoken_path else "" return f"{scheme}{spoken_domain}{spoken_path}".strip() text = re.sub(r"https?://[^\s]+", normalize_url, text) # Normalize bare domains like www.amazon.in, openai.com def normalize_bare_url(match): domain_parts = match.group(0).split(".") return " ଡଟ୍ ".join(" ".join(part.upper()) for part in domain_parts) bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b" text = re.sub( bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE ) # Email normalization text = re.sub( r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b", lambda m: f"{m.group(1)} ଏଟ୍ {m.group(2)} ଡଟ୍ {m.group(3)}", text, ) symbol_map = { "&": " or ", "@": " at ", "%": " percent ", "+": " plus ", "-": " minus ", "=": " equals ", "*": " star ", "#": " hash ", } for sym, word in symbol_map.items(): text = text.replace(sym, word) def expand_acronyms(match): return " ".join(match.group(0)) text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text) has_digits = any(char.isdigit() for char in text) if has_digits: digit_parts = re.findall(r"\d+", text) for part in digit_parts: text = text.replace(part, num2words(part, lang="or")) return text.lower() # %% ../nbs/1b.indic_normalizer.ipynb 27 class BengaliNormalizer(BaseNormalizer): """ Normalizer for the Bengali script. In addition to basic normalization by the super class, * Replaces the composite characters containing nuktas by their decomposed form * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama * Canonicalize two part dependent vowels * replace pipe character '|' by poorna virama character * replace colon ':' by visarga if the colon follows a charcter in this script """ NUKTA = "\u09bc" def __init__( self, lang="bn", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, do_remap_assamese_chars=False, tts_mode=False, ): super(BengaliNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) self.do_remap_assamese_chars = do_remap_assamese_chars self.tts_mode = True def __call__(self, text: str): # common normalization for Indic scripts text = super(BengaliNormalizer, self).normalize(text) # decomposing Nukta based composite characters text = text.replace("\u09dc", "\u09a1" + BengaliNormalizer.NUKTA) text = text.replace("\u09dd", "\u09a2" + BengaliNormalizer.NUKTA) text = text.replace("\u09df", "\u09af" + BengaliNormalizer.NUKTA) if self.remove_nuktas: text = text.replace(BengaliNormalizer.NUKTA, "") if self.do_remap_assamese_chars and self.lang == "as": text = text.replace("\u09f0", "\u09b0") # 'ra' character text = text.replace("\u09f1", "\u09ac") # 'va' character # replace the poorna virama codes specific to script # with generic Indic script codes text = text.replace("\u09e4", "\u0964") text = text.replace("\u09e5", "\u0965") # replace pipe character for poorna virama text = text.replace("\u007c", "\u0964") # replace bengali currency numerator four for poorna virama (it looks similar and is used as a substitute) text = text.replace("\u09f7", "\u0964") # two part dependent vowels text = text.replace("\u09c7\u09be", "\u09cb") text = text.replace("\u09c7\u09d7", "\u09cc") # correct visarge text = re.sub(r"([\u0980-\u09ff]):", "\\1\u0983", text) if self.tts_mode: text = re.sub(r"INR\s+(\d+)", r"রুপি \1", text) text = re.sub(r"Rs\.\s+(\d+)", r"রুপি \1", text) text = re.sub(r"₹\s*(\d+)", r"রুপি \1", text) text = re.sub(r"USD\s+(\d+)", r"ডলার \1", text) text = re.sub(r"\$\s*(\d+)", r"ডলার \1", text) text = re.sub(r"KRW\s+(\d+)", r"কোরিয়ান ওন \1", text) text = re.sub(r"₩\s*(\d+)", r"কোরিয়ান ওন \1", text) # Handle decimal numbers def replace_decimal(match): whole, frac = match.group(1), match.group(2) return f"{whole} পয়েন্ট {frac}" text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text) # Full URL normalization def normalize_url(match): url = match.group(0) scheme = "" if url.startswith("https://"): scheme = "এইচ টি টি পি এস কোলন স্ল্যাশ স্ল্যাশ " url = url[len("https://") :] elif url.startswith("http://"): scheme = "এইচ টি টি পি কোলন স্ল্যাশ স্ল্যাশ " url = url[len("http://") :] parts = url.split("/", 1) domain = parts[0] path = parts[1] if len(parts) > 1 else "" domain_parts = domain.split(".") spoken_domain = " ডট ".join( " ".join(part.upper()) for part in domain_parts ) spoken_path = "" if path: path_parts = path.split("/") spoken_path = " স্ল্যাশ ".join( " ".join(p.upper()) for p in path_parts if p ) spoken_path = " স্ল্যাশ " + spoken_path if spoken_path else "" return f"{scheme}{spoken_domain}{spoken_path}".strip() text = re.sub(r"https?://[^\s]+", normalize_url, text) # Bare domain normalization def normalize_bare_url(match): domain_parts = match.group(0).split(".") return " ডট ".join(" ".join(part.upper()) for part in domain_parts) bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b" text = re.sub( bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE ) # Emails text = re.sub( r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b", lambda m: f"{m.group(1)} অ্যাট {m.group(2)} ডট {m.group(3)}", text, ) # Replace standalone special characters symbol_map = { "&": " or ", "@": " at ", "%": " percent ", "+": " plus ", "-": " minus ", "=": " equals ", "*": " star ", "#": " hash ", } for sym, word in symbol_map.items(): text = text.replace(sym, word) def expand_acronyms(match): return " ".join(match.group(0)) text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text) has_digits = any(char.isdigit() for char in text) if has_digits: digit_parts = re.findall(r"\d+", text) for part in digit_parts: text = text.replace(part, num2words(part, lang="bn")) return text.lower() # %% ../nbs/1b.indic_normalizer.ipynb 29 class TamilNormalizer(BaseNormalizer): """ Normalizer for the Tamil script. In addition to basic normalization by the super class, * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama * canonicalize two-part dependent vowel signs * replace colon ':' by visarga if the colon follows a charcter in this script """ def __init__( self, lang="ta", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, tts_mode=False, ): super(TamilNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) self.tts_mode = tts_mode def __call__(self, text: str): # common normalization for Indic scripts text = super(TamilNormalizer, self).normalize(text) # replace the poorna virama codes specific to script # with generic Indic script codes text = text.replace("\u0be4", "\u0964") text = text.replace("\u0be5", "\u0965") # two part dependent vowels text = text.replace("\u0b92\u0bd7", "\u0b94") text = text.replace("\u0bc6\u0bbe", "\u0bca") text = text.replace("\u0bc7\u0bbe", "\u0bcb") text = text.replace("\u0bc6\u0bd7", "\u0bcc") # correct visarge text = re.sub(r"([\u0b80-\u0bff]):", "\\1\u0b83", text) if self.tts_mode: # Handle currencies text = re.sub(r"INR\s+(\d+)", r"ரூபாய் \1", text) text = re.sub(r"Rs\.\s+(\d+)", r"ரூபாய் \1", text) text = re.sub(r"₹\s*(\d+)", r"ரூபாய் \1", text) text = re.sub(r"USD\s+(\d+)", r"டாலர் \1", text) text = re.sub(r"\$\s*(\d+)", r"டாலர் \1", text) text = re.sub(r"KRW\s+(\d+)", r"கொரிய வான் \1", text) text = re.sub(r"₩\s*(\d+)", r"கொரிய வான் \1", text) # Handle decimal numbers def replace_decimal(match): whole, frac = match.group(1), match.group(2) return f"{whole} பாயிண்ட் {frac}" text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text) # Normalize full URLs def normalize_url(match): url = match.group(0) scheme = "" if url.startswith("https://"): scheme = "எச் டி டி பி எஸ் கோலன் ஸ்லாஷ் ஸ்லாஷ் " url = url[len("https://") :] elif url.startswith("http://"): scheme = "எச் டி டி பி கோலன் ஸ்லாஷ் ஸ்லாஷ் " url = url[len("http://") :] parts = url.split("/", 1) domain = parts[0] path = parts[1] if len(parts) > 1 else "" domain_parts = domain.split(".") spoken_domain = " டாட் ".join( " ".join(part.upper()) for part in domain_parts ) spoken_path = "" if path: path_parts = path.split("/") spoken_path = " ஸ்லாஷ் ".join( " ".join(p.upper()) for p in path_parts if p ) spoken_path = " ஸ்லாஷ் " + spoken_path if spoken_path else "" return f"{scheme}{spoken_domain}{spoken_path}".strip() text = re.sub(r"https?://[^\s]+", normalize_url, text) # Normalize bare domains (www.amazon.in) def normalize_bare_url(match): domain_parts = match.group(0).split(".") return " டாட் ".join(" ".join(part.upper()) for part in domain_parts) bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b" text = re.sub( bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE ) # Emails text = re.sub( r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b", lambda m: f"{m.group(1)} அட் {m.group(2)} டாட் {m.group(3)}", text, ) # Replace standalone special characters symbol_map = { "&": " or ", "@": " at ", "%": " percent ", "+": " plus ", "-": " minus ", "=": " equals ", "*": " star ", "#": " hash ", } for sym, word in symbol_map.items(): text = text.replace(sym, word) def expand_acronyms(match): return " ".join(match.group(0)) text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text) has_digits = any(char.isdigit() for char in text) if has_digits: digit_parts = re.findall(r"\d+", text) for part in digit_parts: text = text.replace(part, num2words(part, lang="ta")) return text.lower() # %% ../nbs/1b.indic_normalizer.ipynb 32 class KannadaNormalizer(BaseNormalizer): """ Normalizer for the Kannada script. In addition to basic normalization by the super class, * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama * canonicalize two-part dependent vowel signs * replace colon ':' by visarga if the colon follows a charcter in this script """ def __init__( self, lang="kn", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, tts_mode=False, ): super(KannadaNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) self.tts_mode = tts_mode def __call__(self, text: str): # common normalization for Indic scripts text = super(KannadaNormalizer, self).normalize(text) # replace the poorna virama codes specific to script # with generic Indic script codes text = text.replace("\u0ce4", "\u0964") text = text.replace("\u0ce5", "\u0965") # dependent vowels text = text.replace("\u0cbf\u0cd5", "\u0cc0") text = text.replace("\u0cc6\u0cd5", "\u0cc7") text = text.replace("\u0cc6\u0cd6", "\u0cc8") text = text.replace("\u0cc6\u0cc2", "\u0cca") text = text.replace("\u0cca\u0cd5", "\u0ccb") # correct visarge text = re.sub(r"([\u0c80-\u0cff]):", "\\1\u0c83", text) if self.tts_mode: # Handle currencies text = re.sub(r"INR\s+(\d+)", r"ರೂಪಾಯಿ \1", text) text = re.sub(r"Rs\.\s+(\d+)", r"ರೂಪಾಯಿ \1", text) text = re.sub(r"₹\s*(\d+)", r"ರೂಪಾಯಿ \1", text) text = re.sub(r"USD\s+(\d+)", r"ಡಾಲರ್ \1", text) text = re.sub(r"\$\s*(\d+)", r"ಡಾಲರ್ \1", text) text = re.sub(r"KRW\s+(\d+)", r"ಕೊರಿಯನ್ ವಾನ್ \1", text) text = re.sub(r"₩\s*(\d+)", r"ಕೊರಿಯನ್ ವಾನ್ \1", text) # Handle decimal numbers def replace_decimal(match): whole, frac = match.group(1), match.group(2) return f"{whole} ಪಾಯಿಂಟ್ {frac}" text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text) # Normalize full URLs def normalize_url(match): url = match.group(0) scheme = "" if url.startswith("https://"): scheme = "ಎಚ್ ಟಿ ಟಿ ಪಿ ಎಸ್ ಕೋಲನ್ ಸ್ಲ್ಯಾಶ್ ಸ್ಲ್ಯಾಶ್ " url = url[len("https://") :] elif url.startswith("http://"): scheme = "ಎಚ್ ಟಿ ಟಿ ಪಿ ಕೋಲನ್ ಸ್ಲ್ಯಾಶ್ ಸ್ಲ್ಯಾಶ್ " url = url[len("http://") :] parts = url.split("/", 1) domain = parts[0] path = parts[1] if len(parts) > 1 else "" domain_parts = domain.split(".") spoken_domain = " ಡಾಟ್ ".join( " ".join(part.upper()) for part in domain_parts ) spoken_path = "" if path: path_parts = path.split("/") spoken_path = " ಸ್ಲ್ಯಾಶ್ ".join( " ".join(p.upper()) for p in path_parts if p ) spoken_path = " ಸ್ಲ್ಯಾಶ್ " + spoken_path if spoken_path else "" return f"{scheme}{spoken_domain}{spoken_path}".strip() text = re.sub(r"https?://[^\s]+", normalize_url, text) # Normalize bare domains like www.amazon.in def normalize_bare_url(match): url = match.group(0) domain_parts = url.split(".") spoken = " ಡಾಟ್ ".join(" ".join(part.upper()) for part in domain_parts) return spoken bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b" text = re.sub( bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE ) # Normalize email text = re.sub( r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b", lambda m: f"{m.group(1)} ಅಟ್ {m.group(2)} ಡಾಟ್ {m.group(3)}", text, ) # Replace standalone special characters symbol_map = { "&": " or ", "@": " at ", "%": " percent ", "+": " plus ", "-": " minus ", "=": " equals ", "*": " star ", "#": " hash ", } for sym, word in symbol_map.items(): text = text.replace(sym, word) def expand_acronyms(match): return " ".join(match.group(0)) text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text) has_digits = any(char.isdigit() for char in text) if has_digits: digit_parts = re.findall(r"\d+", text) for part in digit_parts: text = text.replace(part, num2words(part, lang="kn")) return text.lower() # %% ../nbs/1b.indic_normalizer.ipynb 34 class MalayalamNormalizer(BaseNormalizer): """ Normalizer for the Malayalam script. In addition to basic normalization by the super class, * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama * canonicalize two-part dependent vowel signs * Change from old encoding of chillus (till Unicode 5.0) to new encoding * replace colon ':' by visarga if the colon follows a charcter in this script """ CHILLU_CHAR_MAP = { "\u0d7a": "\u0d23", "\u0d7b": "\u0d28", "\u0d7c": "\u0d30", "\u0d7d": "\u0d32", "\u0d7e": "\u0d33", "\u0d7f": "\u0d15", } def _canonicalize_chillus(self, text): for chillu, char in MalayalamNormalizer.CHILLU_CHAR_MAP.items(): text = text.replace(chillu, "{}\u0d4d".format(char)) return text def _correct_geminated_T(self, text): return text.replace("\u0d31\u0d4d\u0d31", "\u0d1f\u0d4d\u0d1f") def __init__( self, lang="ml", remove_nuktas=False, nasals_mode="do_nothing", do_normalize_chandras=False, do_normalize_vowel_ending=False, do_canonicalize_chillus=False, do_correct_geminated_T=False, tts_mode=False, ): super(MalayalamNormalizer, self).__init__( lang, remove_nuktas, nasals_mode, do_normalize_chandras, do_normalize_vowel_ending, ) self.do_canonicalize_chillus = do_canonicalize_chillus self.do_correct_geminated_T = do_correct_geminated_T self.tts_mode = tts_mode def __call__(self, text: str): # Change from old encoding of chillus (till Unicode 5.0) to new encoding text = text.replace("\u0d23\u0d4d\u200d", "\u0d7a") text = text.replace("\u0d28\u0d4d\u200d", "\u0d7b") text = text.replace("\u0d30\u0d4d\u200d", "\u0d7c") text = text.replace("\u0d32\u0d4d\u200d", "\u0d7d") text = text.replace("\u0d33\u0d4d\u200d", "\u0d7e") text = text.replace("\u0d15\u0d4d\u200d", "\u0d7f") # Normalize chillus if self.do_canonicalize_chillus: text = self._canonicalize_chillus(text) # common normalization for Indic scripts text = super(MalayalamNormalizer, self).normalize(text) # replace the poorna virama codes specific to script # with generic Indic script codes text = text.replace("\u0d64", "\u0964") text = text.replace("\u0d65", "\u0965") # dependent vowels text = text.replace("\u0d46\u0d3e", "\u0d4a") text = text.replace("\u0d47\u0d3e", "\u0d4b") # au forms text = text.replace("\u0d46\u0d57", "\u0d4c") text = text.replace("\u0d57", "\u0d4c") # correct geminated T if self.do_correct_geminated_T: text = self._correct_geminated_T(text) # correct visarga text = re.sub(r"([\u0d00-\u0d7f]):", "\\1\u0d03", text) # remove samvruthokaram text = text.replace("\u0d41\u0d4d", "\u0d4d") if self.tts_mode: # Handle currencies text = re.sub(r"INR\s+(\d+)", r"രൂപ \1", text) text = re.sub(r"Rs\.\s+(\d+)", r"രൂപ \1", text) text = re.sub(r"₹\s*(\d+)", r"രൂപ \1", text) text = re.sub(r"USD\s+(\d+)", r"ഡോളർ \1", text) text = re.sub(r"\$\s*(\d+)", r"ഡോളർ \1", text) text = re.sub(r"KRW\s+(\d+)", r"കൊറിയൻ വോൺ \1", text) text = re.sub(r"₩\s*(\d+)", r"കൊറിയൻ വോൺ \1", text) # Decimal numbers def replace_decimal(match): whole, frac = match.group(1), match.group(2) return f"{whole} പോയിന്റ് {frac}" text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text) # Normalize full URLs (http/https) def normalize_url(match): url = match.group(0) scheme = "" if url.startswith("https://"): scheme = "എച്ച് ടി ടി പി എസ് കോളൺ സ്ലാഷ് സ്ലാഷ് " url = url[len("https://") :] elif url.startswith("http://"): scheme = "എച്ച് ടി ടി പി കോളൺ സ്ലാഷ് സ്ലാഷ് " url = url[len("http://") :] parts = url.split("/", 1) domain = parts[0] path = parts[1] if len(parts) > 1 else "" domain_parts = domain.split(".") spoken_domain = " ഡോട്ട് ".join( " ".join(part.upper()) for part in domain_parts ) spoken_path = "" if path: path_parts = path.split("/") spoken_path = " സ്ലാഷ് ".join( " ".join(p.upper()) for p in path_parts if p ) spoken_path = " സ്ലാഷ് " + spoken_path if spoken_path else "" return f"{scheme}{spoken_domain}{spoken_path}".strip() text = re.sub(r"https?://[^\s]+", normalize_url, text) # Normalize bare domain URLs (like www.amazon.in) def normalize_bare_url(match): url = match.group(0) domain_parts = url.split(".") spoken = " ഡോട്ട് ".join(" ".join(part.upper()) for part in domain_parts) return spoken bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b" text = re.sub( bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE ) # Normalize emails text = re.sub( r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b", lambda m: f"{m.group(1)} അറ്റ് {m.group(2)} ഡോട്ട് {m.group(3)}", text, ) # Replace special characters symbol_map = { "&": " or ", "@": " at ", "%": " percent ", "+": " plus ", "-": " minus ", "=": " equals ", "*": " star ", "#": " hash ", } for sym, word in symbol_map.items(): text = text.replace(sym, word) def expand_acronyms(match): return " ".join(match.group(0)) text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text) has_digits = any(char.isdigit() for char in text) if has_digits: digit_parts = re.findall(r"\d+", text) for part in digit_parts: text = text.replace(part, num2words(part, lang="ml")) return text.lower()