o Kεi{6@s<dZddlmZddlmZddlmZGdddeZdS) z @author:Bengali.AI )print_function) languages)langsc@seZdZ   d+ddZddZddZd d Zd d Zd dZddZ ddZ ddZ ddZ ddZ ddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*ZdS),BaseNormalizerFNcCst|tks Jd|tvsJdt|tksJdt|tks(Jdt||_|durjt|tks;Jdt|dksGJd|D]\}}||jj vs\J|d ||jj vsiJ|d qK|jj |_ |jj |_ ||_|rttt|j td j |_ ttt|j td j |_ |rttt|j |jj |_ ttt|j |jj |_ |j|jd |_|j|j|j|j|jd |_dS)aa initialize a normalizer args: language : language identifier/name type(str) allow_english : allow english letters numbers and punctuations [default:False] type(bool) keep_legacy_symbols : legacy symbols will be considered as valid unicodes[default:False] type(bool) legacy_maps : a dictionay for changing legacy symbols into a more used unicode [default:None] type(dict/None) zlanguage is not string type!!!zLanguage is not availablez1allow_english is not of type boolean [True/False]z7keep_legacy_symbols is not of type boolean [True/False]Nz'legacy_maps is not of type dict or Nonerzlegacy_maps is an empty dictzO is not a legacy symbol.See README.md initialization section for legacy symbolszR is not a valid legacy map.See README.md initialization section for legacy symbolsenglish) LegacySymbolsBrokenDiacritics) BrokenNuktaInvalidUnicodeInvalidConnector FixDiacriticsVowelDiacriticAfterVowel)typestrrkeysboollangdictlenitemslegacy_symbolsusedvalid complex_rootsroots legacy_mapssortedlistsetmapLegacySymbolsfixBrokenDiacriticsword_level_opsfixBrokenNuktacleanInvalidUnicodescleanInvalidConnectorcleanDiacritics#cleanVowelDiacriticComingAfterVoweldecomp_level_ops)selflanguage allow_englishkeep_legacy_symbolsrkvr/L/home/ubuntu/.local/lib/python3.10/site-packages/bnunicodenormalizer/base.py__init__s:    zBaseNormalizer.__init__cCst|jdkr dSdS)NrTF)rdecompr)r/r/r0 checkDecompAszBaseNormalizer.checkDecompcCs&|D] \}}|j|||_qdSN)rwordreplace)r)map_dictr-r.r/r/r0 replaceMapsEszBaseNormalizer.replaceMapscCs(|j|}|j||j|<||j|<dSr5)r2)r)idx1idx2tempr/r/r0swapIdxsIs zBaseNormalizer.swapIdxscCs^dd|jD|_ddd|jD|_|dd|jD|_ddd|jD|_dS)NcSg|]}|dur|qSr5r/.0xr/r/r0 Pz)BaseNormalizer.safeop..cSr>r5r/r?r/r/r0rBQrCcSr>r5r/r?r/r/r0rBTrCcSr>r5r/r?r/r/r0rBUrC)r2join)r)opr/r/r0safeopNs zBaseNormalizer.safeopc sTjjjvrfddtjD}dd|D}g}t|dkrm|^}}t|}d}t||kr\t|}g}|D]}t|t|dkrN|t|O}q:||q:|}t||ks2|tt ||}t|dks"g} |D]+} d} | D] } | j| 7} qw| | | D]} | | dkr| j| <qdj| <qqqddjD_dSdS) z9 creates grapheme root based decomp cs g|] \}}|jjkr|qSr/)r connector)r@irAr3r/r0rBas z9BaseNormalizer.constructComplexDecomp..cSsg|] }|d||dgqS)rr/)r@cidr/r/r0rBcsrrDNcSr>r5r/)r@dr/r/r0rBrC) rrHr2 enumeraterr intersectionappendrr) r)c_idxscompsr_decompfirstrestlfrest2rcombsridxcombrIr/r3r0constructComplexDecomp[sB          z%BaseNormalizer.constructComplexDecompcCs|jdur ||jdSdSr5)rr9r3r/r/r0r s zBaseNormalizer.mapLegacySymbolscCs"|jjdur||jjdSdSr5)r diacritic_mapr9r3r/r/r0r!s z"BaseNormalizer.fixBrokenDiacriticscCs|jjdur?t|jD]5\}}||jjkr>t|dddD] }|j||jjvr=|jj|j||j|<d|j|<nqq dSdS)NrrK)r nukta_maprMr2nuktaranger)r)idxrLcidxr/r/r0r#s   zBaseNormalizer.fixBrokenNuktacCs|r"|jd|jjvr"|jdd|_|r"|jd|jjvs |rD|jd|jjkrD|jdd|_|rD|jd|jjks/|r\t|jD]\}}||jvr[d|j|<qMdSdS)NrrrK)r4r2rinvalid_startsrHrMrr)r`rLr/r/r0r$s  z#BaseNormalizer.cleanInvalidUnicodescCslt|jD].\}}|t|jdkr3||jjkr3|j|d|jjvs.|j|d|jjvr3d|j|<qdSNr)rMr2rrrHinvalid_connectorsrcr/r/r0r%s , z$BaseNormalizer.cleanInvalidConnectorcCsxt|jD]4\}}|t|jdkr9||jjvr9|j|d|jjvr9||j|dkr2d|j|<qd|j|d<qdSrd)rMr2rrvowel_diacriticsrcr/r/r0cleanVowelDiacriticss" z#BaseNormalizer.cleanVowelDiacriticscCsht|jD],\}}|t|jdkr1||jjvr1|j|d|jjvr1||j|dkr1d|j|<qdSrd)rMr2rrconsonant_diacriticsrcr/r/r0cleanConsonantDiacriticss" z'BaseNormalizer.cleanConsonantDiacriticscCs\t|jD]&\}}|t|jdkr+||jjvr+|j|d|jjvr+|||dqdSrd)rMr2rrrhrfr=rcr/r/r0fixDiacriticOrders "z BaseNormalizer.fixDiacriticOrdercCsLt|jD]\}}|dkr#||jjvr#|j|d|jjvr#d|j|<qdS)Nrr)rMr2r diacritics non_charsrcr/r/r0cleanNonCharDiacss " z BaseNormalizer.cleanNonCharDiacscCs4||j||j||j||jdSr5)rGrgrirjrmr3r/r/r0r&s   zBaseNormalizer.cleanDiacriticscCsDt|jD]\}}||jjvr|j|d|jjvrd|j|<qdSrd)rMr2rrfvowelsrcr/r/r0r's " z2BaseNormalizer.cleanVowelDiacriticComingAfterVowelcCst|jD]Z\}}|dkr|j|dvrd|j|<q|t|jdkrA|j|dvrA|j|d|jjgddgvrAd|j|d<|dkr_|j|dvr_|j|ddgddgvr_d|j|<qdS)Nr)‌‍rrorp)rMr2rrrHrcr/r/r0fixNoSpaceChars @2 zBaseNormalizer.fixNoSpaceCharcCs@||j||j||j||j||jdSr5)rGr$r%r&r'rqr3r/r/r0 baseComposes    zBaseNormalizer.baseComposecCsTg}||_t|jtstdt|jddkr'td|jdd||_ |j D]"\}}|j dd}||j dd}||krQ| |||dq/dd |j D|_ |j D]/\}}d|j }|||s|d|j|d Sd|j }||kr| |||dq`||jdd d |j D|_ |j |j|d S) a normalizes a given word args: word : the string to normalize returns: a dictionary- * "given" = provided text * "normalized = normalized text (gives None if during the operation length of the text becomes 0) * "ops" = list of operations (dictionary) that were executed in given text to create normalized text * each dictionary in ops has: * "operation": the name of the operation / problem in given text * "before" : what the text looked like before the specific operation * "after" : what the text looks like after the specific operation z+The provided argument/ word is not a string rziThe provided string has hultiple words.Make sure no space exists in the middle of the text.probable word:rDN) operationbeforeaftercSsg|]}|qSr/r/)r@chr/r/r0rBsz+BaseNormalizer.__call__..) normalizedgivenopscSr>r5r/r?r/r/r0rB/rC)check isinstancer TypeErrorrstripsplit ValueErrorr7r6r"rrOr2r(rErGr4rr)r)r6detailsop_idrFword_before_op word_after_opr/r/r0__call__s6     zBaseNormalizer.__call__)FFN)__name__ __module__ __qualname__r1r4r9r=rGr[r r!r#r$r%rgrirjrmr&r'rqrrrr/r/r/r0r s0 3 /     rN)__doc__ __future__rrrbnunicodenormalizerobjectrr/r/r/r0s