o 'Ni @sddlZddlZddlZddlZddlZddlZddlmZddlm Z ddlm Z ddl m Z Gddde ZGdd d eZed kreejd krTed ed ejdZejd e _dZeejdkrsejddkrqdndZedeeeeZedeeejd ddMZeejddd*ZeD]ZeZe eZ e!e Z"e#d$e"e#dqWdn1swYWddSWddS1swYdSdS)N) lru_cache)langinfo)common)indic_tokenizec@s eZdZdZddZddZdS)MorphAnalyzerIz' Interface for Morph Analyzer cCdSN)wordr r U/home/ubuntu/.local/lib/python3.10/site-packages/indicnlp/morph/unsupervised_morph.py morph_analyzezMorphAnalyzerI.morph_analyzecCrrr )tokensr r r morph_analyze_documentr z%MorphAnalyzerI.morph_analyze_documentN)__name__ __module__ __qualname____doc__r rr r r r rs rc@sDeZdZdZdddZddZddZed d d d Zd dZ dS)UnsupervisedMorphAnalyzerzG Unsupervised Morphological analyser built using Morfessor 2.0 Fc Csr||_||_t}|tjtj ddd ||_ d t t j|dt t j|d|_t|j|_dS)Nmorph morfessorz{}.modelz ^[{}-{}]+$r)lang add_markerr MorfessorIOread_any_modelospathjoinrINDIC_RESOURCES_PATHformat_morfessor_modelchrr SCRIPT_RANGES_script_range_patrecompile_script_check_re)selfrrior r r __init__'s "*z"UnsupervisedMorphAnalyzer.__init__cCsL|jtjvr$|D]}t|tj|jd}|tjkr#|tjkr#dSqdS)NrTF)rrr#ordNUMERIC_OFFSET_STARTNUMERIC_OFFSET_END)r(textcoffsetr r r _contains_number1s z*UnsupervisedMorphAnalyzer._contains_numbercCs|j|o || Sr)r'matchr1)r(r r r r _morphanalysis_needed9sz/UnsupervisedMorphAnalyzer._morphanalysis_neededi@)maxsizecCsXg}||r|j|}|d}|jrddt|D}|S|jr'd|}|g}|S)z Morphanalyzes a single word and returns a list of component morphemes @param word: string input word rcSs,g|]\}}|dkrd|nd|qS)rz{}_S_z{}_R_)r ).0imr r r Hs,z;UnsupervisedMorphAnalyzer.morph_analyze..z{}_E_)r3r!viterbi_segmentr enumerater )r(r m_listvalr r r r <s   z'UnsupervisedMorphAnalyzer.morph_analyzecCs&g}|D] }||}||q|S)a& Morphanalyzes a document, represented as a list of tokens Each word is morphanalyzed and result is a list of morphemes constituting the document @param tokens: string sequence of words @return list of segments in the document after morph analysis )r extend)r(r out_tokenstokenmorphsr r r rWs    z0UnsupervisedMorphAnalyzer.morph_analyze_documentN)F) rrrrr*r1r3rr rr r r r r"s    r__main__zgUsage: python unsupervised_morph.py []rFTrueTzLoading morph analyser for zLoaded morph analyser for rzutf-8w  )%codecssys itertoolsr%rr functoolsrindicnlprrindicnlp.tokenizerobjectrrrlenargvprintexitlanguagerranalyzeropenifileofile readlineslinestriptrivial_tokenizerr morph_tokenswriterr r r r sD(     R          "