o qi%@svdZddlZddlZddlZddddddd d d d d dddddZejdejejBdZ edZ edZ ed d ddeDZedZedZejdejejBdZejdejdZdefd d!Zd5defd#d$Zd5defd%d&Zd5defd'd(Zd5defd)d*Zd6defd+d,Zed-deejDd"Zd6ded.efd/d0Zded.efd1d2Z defd3d4Z!dS)7z Preprocessing utilities NUSDPLNGBPJPYTHBCRCNGNKRWILSVNDEURPHPPYGUAHINR)$uzł£¥u฿u₡u₦u₩u₪u₫u€u₱u₲u₴u₹z\(?:^|(?<=[^\w@.)]))([\w+-](\.(?!\.))?)*?[\w+-]@(?:\w-?)*?\w+(\.([a-z]{2,})){1,3}(?:$|(?=\b))flagszp(?:^|(?<=[^\w)]))(\+?1[ .-]?)?(\(?\d{3}\)?[ .-]?)?(\d{3}[ .-]?\d{4})(\s?(?:ext\.?|[#x-])\s?\d{2,6})?(?:$|(?=\W))u{(?:^|(?<=[^\w,.]))[+–-]?(([1-9]\d{0,2}(,\d{3})+(\.\d*)?)|([1-9]\d{0,2}([ .]\d{3})+(,\d*)?)|(\d*?[.,]\d+)|\d+)(?:$|(?=\b))z({})+|ccs|]}t|VqdSN)reescape.0crO/home/ubuntu/.local/lib/python3.10/site-packages/urduhack/preprocessing/util.py srz((\r\n)|[\n\v])+z (?!\n)\s+a(?:^|(?>> from urduhack.preprocessing import normalize_whitespace >>> text = "عراق اور شام اعلان کیا ہے دونوں جلد اپنے گے؟" >>> normalized_text = normalize_whitespace(text) >>> normalized_text عراق اور شام اعلان کیا ہے دونوں جلد اپنے گے؟  z\n)_NONBREAKING_SPACE_REsub _LINEBREAK_REstripr rrrnormalize_whitespaceGsr'cCst|t||S)u Replace all URLs in ``text`` str with ``replace_with`` str. Args: text (str): ``Urdu`` text replace_with (str): Replace string Returns: str: Returns a ``str`` object replace url with ``replace_with`` text. Examples: >>> from urduhack.preprocessing import replace_urls >>> text = "20 www.gmail.com فیصد" >>> replace_urls(text) '20 فیصد' )_URL_REr# _SHORT_URL_REr replace_withrrr replace_urlsZr-cC t||S)u Replace all emails in ``text`` str with ``replace_with`` str. Args: text (str): ``Urdu`` text replace_with (str): Replace string Returns: str: Returns a ``str`` object replace emails with ``replace_with`` text. Examples: >>> text = "20 gunner@gmail.com فیصد" >>> from urduhack.preprocessing import replace_emails >>> replace_emails(text) ) _EMAIL_REr#r+rrrreplace_emailsls r1cCr/)u Replace all phone numbers in ``text`` str with ``replace_with`` str. Args: text (str): ``Urdu`` text replace_with (str): Replace string Returns: str: Returns a ``str`` object replace number_no with ``replace_with`` text. Examples: >>> from urduhack.preprocessing import replace_numbers >>> text = "20 فیصد" >>> replace_numbers(text) ' فیصد' ) _PHONE_REr#r+rrrreplace_phone_numbers} r3cCr/)ug Replace all numbers in ``text`` str with ``replace_with`` str. Args: text (str): ``Urdu`` text replace_with (str): Replace string Returns: str: Returns a ``str`` object replace number with ``replace_with`` text. Examples: >>> from urduhack.preprocessing import replace_phone_numbers >>> text = "یعنی لائن آف کنٹرول پر فائربندی کا معاہدہ 555-123-4567 میں ہوا تھا" >>> replace_phone_numbers(text) 'یعنی لائن آف کنٹرول پر فائربندی کا معاہدہ میں ہوا تھا' ) _NUMBERS_REr#r+rrrreplace_numbersr4r6cCs6|durtD] \}}|||}q|St||S)uZ Replace all currency symbols in ``text`` str with string specified by ``replace_with`` str. Args: text (str): Raw text replace_with (str): if None (default), replace symbols with their standard 3-letter abbreviations (e.g. '$' with 'USD', '£' with 'GBP'); otherwise, pass in a string with which to replace all symbols (e.g. "*CURRENCY*") Returns: str: Returns a ``str`` object containing normalized text. Examples: >>> from urduhack.preprocessing import replace_currency_symbols >>> text = "یعنی لائن آف کنٹرول پر فائربندی کا معاہدہ 2003 میں ہوا 33$ تھا۔" >>> replace_currency_symbols(text) 'یعنی لائن آف کنٹرول پر فائربندی کا معاہدہ 2003 میں ہوا 33USD تھا۔' N) CURRENCIESitemsreplace _CURRENCY_REr#)r r,keyvaluerrrreplace_currency_symbolss  r=ccs(|]}tt|dr|VqdS)PN) unicodedatacategorychr startswith)rirrrrsreturncCs.|rtjdt|d|tjdS|tS)u, Remove punctuation from ``text`` by removing all instances of ``marks``. Args: text (str): Urdu text marks (str): If specified, remove only the characters in this string, e.g. ``marks=',;:'`` removes commas, semi-colons, and colons. Otherwise, all punctuation marks are removed. Returns: str: returns a ``str`` object containing normalized text. Note: When ``marks=None``, Python's built-in :meth:`str.translate()` is used to remove punctuation; otherwise, a regular expression is used instead. The former's performance is about 5-10x faster. Examples: >>> from urduhack.preprocessing import remove_punctuation >>> output = remove_punctuation("کر ؟ سکتی ہے۔") کر سکتی ہے z[{}]+r(r)rr#formatrUNICODE translatePUNCTUATION_TRANSLATE_UNICODE)r marksrrrremove_punctuations  rJcCsddd|DS)u Remove accents from any accented unicode characters in ``text`` str, either by transforming them into ascii equivalents or removing them entirely. Args: text (str): Urdu text Returns: str Examples: >>> from urduhack.preprocessing import remove_accents >>>text = "دالتِ عظمیٰ درخواست" >>> remove_accents(text) 'دالت عظمی درخواست' r(css|] }t|s|VqdSr)r? combiningrrrrrsz!remove_accents..)joinr&rrrremove_accentsr.rMcCs"d}tdd|D}||S)z Removes ``English`` words and digits from a ``text`` Args: text (str): Urdu text Returns: str: ``str`` object with english alphabets removed >ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890cSsi|]}|dqSrr)rr;rrr sz,remove_english_alphabets..)str maketransrG)r characterstablerrrremove_english_alphabetss  rT)r(r)"__doc__sysr?regexrr7compile IGNORECASErFr0r2r5rErLr:r$r"r)r*rPr'r-r1r3r6r=dictfromkeysrange maxunicoderHrJrMrTrrrrsF          !"