o SiaM@sddlmZmZdZdZddlZddlZddlZddlZddl Z ddl m Z ddl m Z ddlmZdd lTejd krBdd lmZnejZd d ZdZdZeejZee Z!e!"ej#e!$eiZ%da&e'dej(Z)e'dej(Z*e'dej(Z+e'dej(Z,ddZ-Gddde.Z/e/a0d-dd Z1t0j2Z2t0j3Z3t0j4a4t0j5Z5t0j6a6t0j7Z7t0j8Z8t0j9Z9t0j:Z:t0j;Z;t0jZ>t0j?Z?t0j@Z@ddZAddZBddZCddZAdd ZDd!d"ZEd.d%d&ZFd/d'd(ZGd-d)d*ZHd+d,ZIdS)0)absolute_importunicode_literalsz0.42.1MITN)md5)log)finalseg)*nt)movecCstjtjt|SN)ospathnormpathjoingetcwd)rrB/home/ubuntu/.local/lib/python3.10/site-packages/jieba/__init__.pysrzdict.txtz^(.+?)( [0-9]+)?( [a-z]+)?$z [a-zA-Z0-9]u([一-鿕a-zA-Z0-9+#&\._%\-]+)z( |\s)cCst|dSr )default_loggersetLevel) log_levelrrr setLogLevel1rc@seZdZefddZddZeddZd5dd Zd d Z d d Z ddZ ddZ ddZ ddZd6ddZd7ddZddZddZeZeZd d!Zd"d#Zd$d%Zd&d'Zd(d)Zd8d*d+Zd,d-Zd9d.d/Zd:d1d2Zd3d4ZdS); TokenizercCsLt|_|tkr ||_nt||_i|_d|_i|_d|_ d|_ d|_ dS)NrF) threadingRLocklock DEFAULT_DICT dictionary _get_abs_pathFREQtotaluser_word_tag_tab initializedtmp_dir cache_file)selfrrrr__init__7s   zTokenizer.__init__cCs d|jS)Nz)rr'rrr__repr__D zTokenizer.__repr__c Csi}d}t|}t|dD]M\}}z9|d}|ddd\}}t|}|||<||7}tt|D]}|d|d} | |vrHd|| <q6Wq tyZtd|||fw| ||fS)Nrrutf-8 z-invalid dictionary entry in %s at Line %s: %s) resolve_filename enumeratestripdecodesplitintxrangelen ValueErrorclose) flfreqltotalf_namelinenolinewordfreqchwfragrrr gen_pfdictGs.  zTokenizer.gen_pfdictNc Cs|rt|}|j|kr|jrdS||_d|_n|j}|jRzt|Wdn1s/wYWn ty>Ynw|jrK WddStd|pQdt}|j r_|j }n|t krfd}n dt | dd }tj|jp{t|}tj|}d}tj|r|t kstj|tj|krtd |z"t|d }t|\|_|_Wdn1swYd}Wn tyd}Ynw|rUt|t}|t|<|Z| |!\|_|_td |z0tj"|d \} } t#| d } t$|j|jf| Wdn 1swYt%| |Wnty4t&dYnwWdn 1s@wYzt|=Wn tyTYnwd|_tdt|tdWddS1stwYdS)NFz Building prefix dict from %s ...zthe default dictionaryz jieba.cachezjieba.u%s.cacher,replaceTzLoading model from cache %srbzDumping model to file cache %s)dirwbzDump cache file failed.z Loading model cost %.3f seconds.z(Prefix dict has been built successfully.)'r rr$r DICT_WRITINGKeyErrorrdebugtimer&rrencode hexdigestr rrr%tempfile gettempdirdirnameisfilegetmtimeopenmarshalloadr!r" ExceptiongetrrrC get_dict_filemkstempfdopendump _replace_file exception) r'rabs_patht1r&tmpdirload_from_cache_failcfwlockfdfpathtemp_cache_filerrr initialize]s           $zTokenizer.initializecCs|js |dSdSr )r$rgr)rrrcheck_initializeds zTokenizer.check_initializedcsZt}d|<tjt|dddD]tfdd|D<qdS)N)rrrc3sH|]}tj|dpd|dd|fVqdS)rrN)rr!rW).0xidxlogtotalrouter'sentencerr s z!Tokenizer.calc..)r6rr"r5max)r'rpDAGroNrrlrcalcs zTokenizer.calccCs|i}t|}t|D]=}g}|}||}||kr@||jvr@|j|r+|||d7}|||d}||kr@||jvs!|sG|||||<q|S)Nr)rhr6r5r!append)r'rprsrtktmplistifragrrrget_DAGs"     zTokenizer.get_DAGc cs||}d}d}d}t|D]Z\}}|dkr$t||s$d}|Vt|dkrV||krV|||dd}t|rJ|dkrFd}|}n||7}|dkrQ|V|d}q|D]} | |kri||| dV| }qXq|dkrt|VdSdS)Nrirr)r{ iteritemsre_engmatchr6) r'rpdagold_jeng_scaneng_bufrwLr?jrrr __cut_alls8    zTokenizer.__cut_allc cs||}i}||||d}t|}d}||krK||dd}|||}t|r;t|dkr;||7}|}n |rB|Vd}|V|}||ks|rT|Vd}dSdSNrr|r)r{rur6r~r) r'rprsrorkrtbufyl_wordrrr__cut_DAG_NO_HMMs.   zTokenizer.__cut_DAG_NO_HMMc cs.||}i}||||d}d}t|}||kri||dd}|||}||dkr4||7}n/|r`t|dkrB|Vd}n|j|sVt|} | D]} | VqOn|D]} | VqXd}|V|}||ks|rt|dkrv|VdS|j|st|} | D]} | VqdS|D]} | VqdSdSr)r{rur6r!rWrcut) r'rprsrorkrrtrr recognizedtelemrrr __cut_DAGsN           zTokenizer.__cut_DAGFTccstd}t|}|r6|r6|dust|dkrdSddlmm}||}|D] }|dur0q)|Vq)dSt} t} |r@|j } n |rF|j } n|j } | |} | D]6} | sUqP| | re| | D]}|Vq^qP| | }|D]}| |rw|Vql|s|D]}|Vq{ql|VqlqPdS)ab The main function that segments an entire sentence that contains Chinese characters into separated words. Parameter: - sentence: The str(unicode) to be segmented. - cut_all: Model type. True for full pattern, False for accurate pattern. - HMM: Whether to use the Hidden Markov Model. is_paddle_installedNr)check_paddle_install strdecoder6jieba.lac_small.predict lac_smallpredictget_sentre_han_defaultre_skip_default_Tokenizer__cut_all_Tokenizer__cut_DAG_Tokenizer__cut_DAG_NO_HMMr3r)r'rpcut_allHMM use_paddlerrresultssentre_hanre_skip cut_blockblocksblkr?tmprkxxrrrr!sN       z Tokenizer.cutccs|j||d}|D]I}t|dkr.tt|dD]}|||d}|j|r-|Vqt|dkrPtt|dD]}|||d}|j|rO|Vq<|Vq dS)z8 Finer segmentation for search engines. rr.rN)rr6r5r!rW)r'rprwordswrygram2gram3rrrcut_for_searchRs"    zTokenizer.cut_for_searchcOt|j|i|Sr )listrr'argskwargsrrrlcutdzTokenizer.lcutcOrr )rrrrrrlcut_for_searchgrzTokenizer.lcut_for_searchcCs||ddSNFrr'rprrr _lcut_no_hmmmrzTokenizer._lcut_no_hmmcC ||dSNTrrrrr _lcut_allp zTokenizer._lcut_allcCrr)rrrrr_lcut_for_search_no_hmmsrz!Tokenizer._lcut_for_search_no_hmmcCs|jtkr ttSt|jdS)NrE)rrget_module_resDEFAULT_DICT_NAMErSr)rrrrXvs  zTokenizer.get_dict_filec Cs|t|tr|}t|d}nt|}t|dD]I\}}|}t|ts?z |d d}Wn t y>t d|w|sBqt |\}}}|durT|}|dur\|}||||qdS)a Load personalized dict to improve detect rate. Parameter: - f : A plain text file contains words and their ocurrences. Can be a file-like object, or the path of the dictionary file, whose encoding must be utf-8. Structure of dict file: word1 freq1 word_type1 word2 freq2 word_type2 ... Word type may be ignored rErr,uz dictionary file %s must be utf-8N)rh isinstance string_typesrSr/r0r1 text_typer2lstripUnicodeDecodeErrorr7 re_userdictrgroupsadd_word) r'r9r<r=lnr>r?r@tagrrr load_userdict|s,     zTokenizer.load_userdictcCs|t|}|durt|n||d}||j|<|j|7_|r)||j|<tt|D]}|d|d}||jvrCd|j|<q/|dkrOt |dSdS)z Add a word to dictionary. freq and tag can be omitted, freq defaults to be a calculated value that ensures the word can be cut out. NFrr) rhrr4 suggest_freqr!r"r#r5r6radd_force_split)r'r?r@rrArBrrrrs    zTokenizer.add_wordcCs||ddS)z: Convenient function for deleting a word. rN)r)r'r?rrrdel_wordszTokenizer.del_wordcCs|t|j}d}t|tr9|}|j|ddD] }||j|d|9}qtt ||jd|j|d}n+t t t |}d |}|D] }||j|d|9}qGtt ||j|j|d}|rl||||S)a Suggest word frequency to force the characters in a word to be joined or splitted. Parameter: - segment : The segments that the word is expected to be cut into, If the word should be treated as a whole, use a str. - tune : If True, tune the word frequency. Note that HMM may affect the final result. If the result doesn't change, set HMM=False. rFrr|r)rhfloatr"rrrr!rWrrr4tuplemaprrminr)r'segmenttuneftotalr@r?segrrrrs   $  zTokenizer.suggest_freqdefaultc cs<t|ts tdd}|dkr,|j||dD]}t|}||||fV||7}qdS|j||dD]h}t|}t|dkrdtt|dD]}|||d}|j|rc|||||dfVqGt|dkrtt|dD]}|||d} |j| r| ||||dfVqr||||fV||7}q3dS) a% Tokenize a sentence and yields tuples of (word, start, end) Parameter: - sentence: the str(unicode) to be segmented. - mode: "default" or "search", "search" is for finer segmentation. - HMM: whether to use the Hidden Markov Model. z-jieba: the input parameter should be unicode.rrrr.rrN)rrr7rr6r5r!rW) r'unicode_sentencemoderstartrwidthryrrrrrtokenizes6       zTokenizer.tokenizecCsX|jt|}tj|std|||_d|_WddS1s%wYdS)Nzjieba: file does not exist: F)rr r rrQrVrr$)r'dictionary_pathr^rrrset_dictionarys  "zTokenizer.set_dictionaryr )FTFT)NN)F)rT)__name__ __module__ __qualname__rr(r* staticmethodrCrgrhrur{rrrrrrr_lcut_lcut_for_searchrrrrXrrrrrrrrrrr5s8    K ( 1 &   !rcCstj||Sr )dtr!rW)rwdrrrr scC t|Sr rrsrrrrr+rcCrr )rrrrrrr"r+rcCrr )rrrrrrr&r+rcCrr rrrrrr*r+cCrr )rrrrrrr.r+rcCrr )rrrrrrr2r+rFTccs^t|d}|rtt|}n|rtt|}ntt|}|D] }|D]}|Vq&q"dSr)r splitlinespoolrrrr)rprrpartsresultrrrrr_pcut6s rccsLt|d}|rtt|}ntt|}|D] }|D]}|VqqdSr)rrrrrr)rprrrrrrrr_pcut_for_searchCs rcCsTddlm}tjdkrtdddlm}t|dur |}||at a t a dS)z Change the module's `cut` and `cut_for_search` functions to the parallel version. Note that this only works using dt, custom Tokenizer instances are not supported. r) cpu_countr z/jieba: parallel mode only supports posix system)PoolN) multiprocessingrr nameNotImplementedErrorrrrhrrrrr) processnumrrrrrenable_parallelNs  rcCs trtdatjatjadSr )rr8rrrrrrrdisable_paralleles  rr )FTr)J __future__rr __version__ __license__rTrerNrrKhashlibrmathrr|r_compatr rshutilr r\renamer rrlogging StreamHandlersysstderr log_console getLoggerrrrDEBUG addHandlerrHrcompileUrr~rrrobjectrrget_FREQrrurrrrrr{rXrgrrrrr#rrrrrrrrrrrrrsx        V