o ik@sddlZddlZddlmZddlZeejejdZeejZeej Z eej Z Gddde ZddZdd Zejejejed ZGd d d e ZdS) N)anyascii_c@s"eZdZdZdddZddZdS)TSResultmatchnormstartendcaseis_exactNcCs(||_||_||_||_||_||_dSNr)selfrrrr r exactrG/home/ubuntu/.local/lib/python3.10/site-packages/textsearch/__init__.py__init__s  zTSResult.__init__cs&djjdfddjDS)Nz{}({}), c s"g|] }d|tt|qS)z{}={})formatreprgetattr.0xr rr s"z%TSResult.__repr__..)r __class____name__join __slots__rrrr__repr__szTSResult.__repr__)NN)r __module__ __qualname__rrrrrrrr s  rcCs|d|ddS)Nr)upperlower)krrrto_sentence_casesr&cCsD||krdS||krdS||krdS|t|kr dSdS)Nr#titler$sentmixed)r#r'r$r&)wordrrrdetermine_case"s    r+)r#r$r'r(c@s4eZdZeeddfddZddZddZd d Zd d Zd dZ dIddZ ddZ dJddZ ddZ ddZddZddZdd Zd!d"ZdJd#d$Zd%d&Zd'd(Zd)d*Zd+d,Zd-d.ZdKd/d0ZdKd1d2ZdLd3d4Zd5d6Zd7d8Zd9d:Zd;d<Zd=d>Z d?d@Z!dAdBZ"dCdDZ#dEdFZ$dGdHZ%dS)M TextSearchFNc Ksd}||vr td|d} t|tr|| vrtd| |ddtttttfvr1td| t |_ ||_ |dkr?|nt |_||_|j dv|_|pQt|_|pWt|_d|_||_|pbg|_i|_d S) ul TextSearch is built on a C implementation of Aho-Corasick available in the ahocorasick[0] lib. It mainly helps with providing convenience for NLP / text search related tasks. For example, it will help find tokens by default only if it is a full word match (and not a sub-match). Though this can be adapted. Arguments: case: one of "ignore", "insensitive", "sensitive", "smart" - ignore: converts both sought words and to-be-searched to lower case before matching. Text matches will always be returned in lowercase. - insensitive: converts both sought words and to-be-searched to lower case before matching. However: it will return the original casing as it uses the position found. - sensitive: does not do any conversion, will only match on exact words added. - smart: takes an input `k` and also adds k.title(), k.upper() and k.lower(). Matches sensitively. returns: one of 'match', 'norm', 'object' (becomes TSResult) or a custom class See the examples! - match: returns the sought key when a match occurs - norm: returns the associated (usually normalized) value when a key match occurs - "object": convenient object for working with matches, e.g.: TSResult(match='HI', norm='greeting', start=0, end=2, case='upper', is_exact=True) - class: bring your own class that will get instantiated like: MyClass(**{"match": k, "norm": v, "case": "lower", "exact": False, start: 0, end: 1}) Trick: to get json-serializable results, use `dict`. left_bound_chars (set(str)): Characters that will determine the left-side boundary check in findall/replace. Defaults to set([A-Za-z0-9_]) right_bound_chars (set(str)): Characters that will determine the right-side boundary check in findall/replace Defaults to set([A-Za-z0-9_]) replace_foreign_chars (default=False): replaces 'á' with 'a' for example in both input and target. Adds a roughly 15% slowdown. handlers (list): provides a way to add hooks to matches. Currently only used when left and/or right bound chars are set. Regex can only be used when using norm The default handler that gets added in any case will check boundaries. Check how to conveniently add regex at the `add_regex_handler` function. Default: (False, True, self.bounds_check) - The first argument should be the normalized tag to fire on. - The second argument should be whether to keep the result - The third should be the handler function, that takes the arguments: - text: the original sentence/document text - start: the starting position of said string - stop: the ending position of said string - norm: the normalized result found Should return: - start: In case start position should change it is possible - stop: In case end position should change it is possible - norm: the new returned item. In case this is None, it will be removed Custom example: >>> def custom_handler(text, start, stop, norm): >>> return start, stop, text[start:stop] + " is OK" >>> ts = TextSearch("ignore", "norm", handlers=[("HI", True, custom_handler)]) >>> ts.add("hi", "HI") >>> ts.findall("hi HI") ['hi is OK', 'HI is OK'] Examples: >>> from textsearch import TextSearch >>> ts = TextSearch(case="ignore", returns="match") >>> ts.add("hi") >>> ts.findall("hello, hi") ["hi"] >>> ts = TextSearch(case="ignore", returns="norm") >>> ts.add("hi", "greeting") >>> ts.add("hello", "greeting") >>> ts.findall("hello, hi") ["greeting", "greeting"] >>> ts = TextSearch(case="ignore", returns="match") >>> ts.add(["hi", "bye"]) >>> ts.findall("hi! bye! HI") ["hi", "bye", "hi"] >>> ts = TextSearch(case="insensitive", returns="match") >>> ts.add(["hi", "bye"]) >>> ts.findall("hi! bye! HI") ["hi", "bye", "HI"] >>> ts = TextSearch("sensitive", "object") >>> ts.add("HI") >>> ts.findall("hi") [] >>> ts.findall("HI") [TSResult(match='HI', norm='HI', start=0, end=2, case='upper', is_exact=True)] >>> ts = TextSearch("sensitive", dict) >>> ts.add("hI") >>> ts.findall("hI") [{'case': 'mixed', 'end': 2, 'exact': True, 'match': 'hI', 'norm': 'hI', 'start': 0}] Notes: [0]: https://github.com/WojciechMula/pyahocorasick )ignore insensitive sensitivesmartz#argument 'case' must be one of {!r})rrobjectz/argument 'returns' must be one of {!r} or classTFr1)r-r.N) ValueErrorr isinstancestrboolintlisttupleset ahocorasick Automaton automatonr rreturnsget_extraction_fn extract_fn_ignore_case_in_searchleft_bound_charsright_bound_chars words_changedreplace_foreign_charshandlers _root_dict) r r r=rArBrDrEkwargs case_opts returns_optsrrrr8s&h       zTextSearch.__init__cCs~t|ts td||}||j||jg}t}|j|jD]}|d|vr9||||dq%||_|S)Nz8Can only be merged with another TextSearch derived class) r3r, TypeErrorto_tsaddrFr9rEappend)r othertsrEseenrrrr__add__s      zTextSearch.__add__cCs|gdd|dS)N)zhttp://zhttps://zwww.z[^ ]+add_regex_handlerr keep_resultrrradd_http_handlerszTextSearch.add_http_handlercCs||||dSr )add_tweet_hashtagadd_tweet_mentionrUrrradd_twitter_handlers zTextSearch.add_twitter_handlercC|dgd|dS)N#z[a-zA-Z][^ !$%^&*+.]+rSrUrrrrXzTextSearch.add_tweet_hashtagcCr[)N@z[a-zA-Z][a-zA-Z0-9_]+rSrUrrrrYr]zTextSearch.add_tweet_mentionTrc Csj|pdtdd|dp|d}|D]}|||q|r#|jn|j} |j||| |||fdS)a Allows adding a regex that should hit when a prefix or postfix gets found. words: list of words that when hit trigger the regex handler. regex: the regex pattern that gets applied to a matched word. NOTE: Already assumes the words to occur at either the start or the end of the match! keep_result: whether to throw away the result when found. prefix: - True means from the start of the match to the end of the whole string - False means means from the start of the whole string to the end of the match name: give a name for the token (used internally, only set to avoid a clash). By default would choose the first of the words, stripped of special chars. return_value: either a fixed value when set, otherwise the regex match. Let's explain this example ts.add_regex_handler(["http://", "https://", "www."], "[^ ]+", keep_result=True) Will match on http://, https://, www. and try the regex '[^ ]+' afterwards, matching urls. Warning: better not to try to use regex with too short of a prefix, e.g. just a single letter. $z [^a-zA-Z]rN)resubr#add_oneprefix_regex_handlerpostfix_regex_handlerrErN) r wordsregexrVprefixname return_valueflagsr*handlerrrrrTs &zTextSearch.add_regex_handlercstd|j}|jjdkrfdd|jD}|di|}nfdd|jD}|jdi|}|||S)N)r<r@rCr?rFtypeci|] \}}|vr||qSrrrr%v block_keysrr z$TextSearch.to_ts..crnrrrorqrrrsrtr)rFrr__dict__itemsrM)r rPdatainitsrrqrrLs  zTextSearch.to_tscCsdt|ttfr|D]}|||q dSt|tr*|D] \}}|||qdS|||dSr )r3r9r7rcdictrv)r r%rprkkvvrrrrMs zTextSearch.addcCsP|jdkr|j|||fdS|jdkr$|j|||fdStd)NrrzLignore only returns a match or normalized value. Maybe you want insensitive?)r=r<add_wordr$r2r r%rplengthrrr add_ignore s  zTextSearch.add_ignorecCsh|jdkr|j|||fdS|jdkr$|j||dfdS|j|||ddfdS)NrrrJFrr)r=r<r|r$r}rrradd_insensitives   zTextSearch.add_insensitivecCs||jvrdS|j|=d|_|jrt|}|jdkrB|j||j||j||j| |jt |dS|jdkrO|j|dS|j|dS)z; Remove k from known words. Takes into account the casing. FTr0r/) rFrCrDrr r< remove_wordr$r'r#r&)r r%rrrremove"s$    zTextSearch.removecCst|}|jdr |n|}|jdr|d|n|}|j|||f|jdkr,dSt|| | t |ggdD]\}}||ksL||jvrMq?|j|||fq?dS)Nr_caserr0r#r'r$r() r+r= startswithendswithr<r|r zipr#r'r$r&)r r%rpr~r text_valueresultkeyrrradd_sensitive_string6s  zTextSearch.add_sensitive_stringcCst|}|||dd}|j|||f|jdkrdSt|||t|ggdD]\}}||ks;||jvrs"z%TextSearch.findall..cSs g|] }|ddur|dqS)Nrrrrrrs z&TextSearch.findall..) rrDrr@r$rErr<rrNmaxnext enumerate)r rkeywords current_stoprrErr~rrrrVrlrkeep_up_to_indrrrfindallsH      zTextSearch.findallcsjr tg}jrn}js-js-js-fddj |DSjddj fg}j |D];\}\}}|durHq=||d}|d} |D]#\} } } | r`| |ur`qT| || |\} } }| smqT|durrqT| |qTq=dd|DS)rcs0g|]\}\}}||d|d|qS)r")r?)rrr~rr rrrrs z/TextSearch.find_overlapping..FTNr"cSsg|]}|dur|qSr rrrrrrs) rrDrr@r$rErArBr<rrrN)r rrrrErr~rrrrrVrlrrrrrfind_overlappings4      zTextSearch.find_overlappingc"tj||dfdd}|S)Nrkcs|dkr||djvr||dfS||d}|rB||}p*|||}tjts5d|i}||||}|||fS||dfS)Nrr"r)rArr r3r=r4r?rrrrreg_resrvrgrjr rr regex_handlers     z6TextSearch.prefix_regex_handler..regex_handlerracompiler rrjrkrrrrrdzTextSearch.prefix_regex_handlercr)Nrcst||kr||jvr||dfS|d|}|r@|}p(|||}tjts3d|i}||||}|||fS||dfS)Nr)rrBsearchrr3r=r4r?rrrrrs    z7TextSearch.postfix_regex_handler..regex_handlerrrrrrre rz TextSearch.postfix_regex_handlercCs|jdkrt|jstd||jrt|}dg}d}|jr%|n|}|jdd|j fg}|j |D]n\}\}} ||d} |d} |D][\} } }| rU| | urUqI||| | | \} } }|duren@| sin<| |kr| }|| | ||| | |ff}| |qI| | |dd krt || }|| | ||| | |ff}||d<qIq7| dt|dd fd }t|dd|ddD]&\\}}}}\}}}}t|dtr|dn|dj} ||||| 7}q|r|d d |ddDfS|S)z Replaces known words in text. text: str returns: replaced str If return_entities=True, returns: replaced str, list of matches rzno idea how i would do that)NNrr`r`rJFTr"Nrrr`cSsg|]}|dqS)rJrrrrrrJsz&TextSearch.replace..)r=callabler2rrDrr@r$rErr<rrNrrrr3r4r)r rreturn_entitiesrrrrErr~rrrrrVrlrtext_rstart1stop1result1start2stop2result2rrrreplacesN     zTextSearch.replacecCs|Sr r)r start_indexr rrrrr extract_strMszTextSearch.extract_strcCs|||}|Sr rr rr rrrrrrextract_insensitive_matchPs z$TextSearch.extract_insensitive_matchcCs$t|||}t|dd|S)NcSs|Sr r)rrrrVsz5TextSearch.extract_insensitive_norm..)r+case_fnget)r rr rrr rrrextract_insensitive_normTsz#TextSearch.extract_insensitive_normcCs@|||}||d<t||d<||d<||d<|jdi|S)Nrr rr r)r+r=rrrrextract_insensitive_objectXs  z%TextSearch.extract_insensitive_objectcCs ||d<||d<|jdi|S)Nrr r)r=)r rr rrrrrextract_object`szTextSearch.extract_objectcCs\|jdkr|jdkr|j}|S|jdkr|j}|S|j}|St|jtr)|j}|S|j}|S)Nr.rr) r r=rrrr3r4rr)r r?rrrr>es    zTextSearch.get_extraction_fncCs t|jSr )rr<rrrr__len__s zTextSearch.__len__cCs |jSr )r<__iter__rrrrrvrzTextSearch.__iter__cCs6|jrt|}|js|jdkr||jvS||jvS)Nr0)rDrr@r r$r<)r rrrr __contains__ys  zTextSearch.__contains__cCszdt|g}|jtkr|dd|j|jtkr&|dd|j|jr.|dd|jj|j |j d |S)Nz num_items={}z{}={!r}rArBzreplace_foreign_chars=Truez{}(case={!r}, returns={!r}, {})r) rrrAALPHANUMrNrBrDrrr r=r)r srrrrs   zTextSearch.__repr__)TNNrr )r)F)&rr r!rrrRrWrZrXrYrTrLrMrrrrrrrcrrrrrrdrerrrrrrr>rrrrrrrrr,7sP       ?  . r,)rastringrr:r9digits ascii_lettersrALPHAascii_lowercase ALPHA_LOWERascii_uppercase ALPHA_UPPERr1rr&r+r4r#r$r'rr,rrrrs"