o }oig@sddlZddlZddlZddlmZmZmZddlmZm Z ddl m Z ddl m Z ddlmZddlmZmZmZmZmZmZmZddlmZd Zz ddlZed ZWneee fydd ZYnwzddl!Z!d Z"Wn eefyzd Z"Ynwzddl#Z#d Z$Wn eefyd Z$Ynwd Z%d Z&dZ'ddZ(ddZ)ddZ*ddZ+ddZ,ddZ-e GdddZ.GdddZ/dnde0d e1d!d"fd#d$Z2 % & 'dod(d"d)ee0d*e3d+e3d,e0d!e4f d-d.Z5d'd d/e6d0fd1d2d3ee4e0fd4e0d5e1d6e4d7e0d!e/fd8d9Z7d:e/d!e/fd;d<Z8 dnd:e/d e1d!d"fd=d>Z9 dpd?e0d@ee0e4fdAe1d e1d!d"f dBdCZ:dnd@ee0e4fdAe1d!d"fdDdEZ;dnd@ee0e4fdAe1d!d"fdFdGZGdUdVdVe Z?GdWdXdXeZ@GdYdZdZeZAGd[d\d\eAZBd!e1fd]d^ZCd_eee0fd`e3d!ee0ee4e4fffdadbZD cdrddededfe3d!dgfdhdiZE dsdjeee0fd3eee4e0fdkeee4e0fd!ee0eBffdldmZFdS)tN)ABCabstractmethodabstractproperty) defaultdict namedtuple) dataclass)Enum)Path)AnyDictList NamedTupleOptionalTupleUnion)loggingu‡TFzkaldifst is not installed or is installed incorrectly. please run `pip install kaldifst` or `bash scripts/installers/install_riva_decoder.sh` to install.z_graphviz is not installed. please run `bash scripts/installers/install_graphviz.sh` to install.z{kaldilm is not installed. please run `pip install kaldilm` or `bash scripts/installers/install_riva_decoder.sh` to install.cCtdurttdSNF)_KALDIFST_AVAILABLE ImportErrorKALDIFST_INSTALLATION_MESSAGErr_/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/asr/parts/utils/wfst_utils.py_kaldifst_maybe_raiseKrcC ttSzUImport helper function that returns kaldifst package or raises ImportError exception.)rkaldifstrrrrkaldifst_importerPrcCrr)_GRAPHVIZ_AVAILABLErGRAPHVIZ_INSTALLATION_MESSAGErrrr_graphviz_maybe_raiseVrr"cCr)zUImport helper function that returns graphviz package or raises ImportError exception.)r"graphvizrrrrgraphviz_importer[rr$cCrr)_KALDILM_AVAILABLErKALDILM_INSTALLATION_MESSAGErrrr_kaldilm_maybe_raisearr'cCrr)r'kaldilmrrrrkaldilm_importerfrr)c@s&eZdZUdZeed<dZeed<dS) LexiconUnitzhA dataclass encapsulating the name of the language unit (e.g. wordpiece) and its mark (e.g. word begin).namemarkN)__name__ __module__ __qualname____doc__str__annotations__r-rrrrr*ls r*c @seZdZedfdeeeeefdeeee feee ffdeeee feee ffde fddZ de e ee ffd d Z d d Zedeefd dZdS)Lexicon^#\d+$wordid2tokenidid2wordid2tokendisambig_patterncstt|dt }|rdd|Dn|_ddjD_tt|dt }|rz$Lexicon.__init__..cSi|]\}}|j|qSrr+r<rrrr@cSr:rr;r<rrrr@rAcSrBrrCr<rrrr@rDcsg|] }fdd|DqS)csg|]}j|jqSr)r8r+r=iselfrr rDz/Lexicon.__init__...r)r=vprGrrrIz$Lexicon.__init__..FdisambigepsilonTN) isinstancelistvaluesr*itemsr8token2idr7word2idr6rr+ word2tokensr9 has_epsilon_default_disambig_mark_default_epsilon_mark_default_epsilon_namematchr- startswithmax_disambig_id num_disambigs)rHr6r7r8r9is_id2token_stris_id2word_strrWr>r?r^r_rFsrrGr__init__usP(    zLexicon.__init__returnccs.|jD]\}}|D]}||fVq qdSN)r6rT)rHwordid tokenid_listtokenidsrrr__iter__s  zLexicon.__iter__cCs t|jSre)r2rWrGrrr__str__s zLexicon.__str__cCsJg}|jD]\}}|j|js|jr|dkr||q||S)zXReturn a list of token IDs excluding those from disambiguation symbols. r)r8rTr-r]rZrXappendsort)rHansrFrbrrr token_idss zLexicon.token_idsN)r.r/r0recompiler intr rr2r*rcrrirjpropertyrnrrrrr4ts Ar4lm_pathattach_symbol_tablerdkaldifst.StdVectorFstc CstttZ}tj|d}tj|d}tj||d|dt j |}|r]t }t |dd}|D]}|\} } |j| t| dq7Wdn1sUwY||_Wdn1sgwYt j|d d |S) a  Compiles an ARPA LM file into a grammar WFST (G.fst). Args: lm_path: Path to the ARPA LM file. attach_symbol_table: Whether to attach the words for indices of the returned WFST. Returns: Kaldi-type grammar WFST. z output.fstz words.txt#0) input_arpa output_fstdisambig_symbolwrite_symbol_tableutf-8encodingsymbolkeyNilabel sort_type)rr'tempfileTemporaryDirectoryospathjoinr(arpa2fstr StdVectorFstread SymbolTableopenstripsplit add_symbolrqoutput_symbolsarcsort) rsrt tempdirnamerx words_txtGosymflinewrFrrrrs2  r@@g_fsttokens word_weighttoken_unigram_weight token_oovc Cstd}t||jjd|jdfvsJ|j}d}|j||| }|j |tj ||||dd|j |tj ||d|dd|d} |D]%} | |krp|j |tj | | ||dd|j| t || | d7} qK|S)a Adds special words representing individual tokens (tokenwords). In-place operation. Args: g_fst: Kaldi-type grammar WFST. Will be augmented with the tokenwords. tokens: Token vocabulary. word_weight: The weight of an Out Of Vocabulary (OOV) word emission. token_unigram_weight: The weight of a tokenword emission. token_oov: OOV token. Returns: The id of the tokenword disambiguation token. rrv#1rolabelweight nextstatestatearcrP) rr ArcIteratorvaluerrfind available_keyr add_stateadd_arcStdArcTW_BREAK) rrrrr unigram_statetokenword_disambig_idtokenword_disambigtokenword_statelabeltrrradd_tokenwords_sR$    rrLr5 tokenizer TokenizerSpecr7oov add_epsilonfirst_tokenword_idr9cs6dd|D}d}dd|dk}|r8gg} } |D]\} } | r(q| |kr1| | n| | qnfdd|Dg} } |j| td } t|t| |fd dD}||}t|d ||<t | }td d ||d<t|d||d<|rtd||d<|rtdd|d<ddt |dddD}|r| | 7} | fdd| D7} t t }t| | D]#\}}|dr|dkrq|dkrq|||fdd|Dqt|||}d|j|j|_|rd|j|j_| D] }d|j|j|_q |S)a( Generate a Lexicon using a SentencePiece tokenizer. Args: tokenizer: NeMo SentencePiece tokenizer. id2word: Word index. Mapping from word_id to word_str. oov: Out Of Vocabulary word in lexicon. Returns: Lexicon object. cSsi|]\}}||qSrrr<rrrr@]z2generate_lexicon_sentencepiece..rvru▁rLcsg|] }|s|qSr)r\)r=r?)r9rrrIjrAz2generate_lexicon_sentencepiece..)out_typecs.i|]\}}|t||rdndqS)beginr,)r*r]r<)maybe_subtract_oneword_begin_markrrr@ssunkblankrPdisambig_backoffdisambig_tokenwordrOrNrcSsi|]\}}||qSrrr<rrrr@rcSs|dS)Nrr)itemrrrsz0generate_lexicon_sentencepiece..)rcs(g|]}|tgqSr)rstripr)r=tw) maybe_add_onervocabrrrIs(r?words_piece_idsr8unk_id max_token_idr6word piece_idslexiconrr)r9rrrrrrgenerate_lexicon_sentencepieceDs`     " rrcCsd|jv}|r|jdd}|jd}n|jd}tt}|D]\}}|t|d7<qtt}|D]!\}}|r?||kr?q4|}||rUd|t|<||sIq4tt} |j} |j } |d} | d} tt}|D]i\}}t|}t |dksJ||dkr||dks|r||kr| | |qp||}|dkr| }| }n t| |j dd}|| kr|} t| d}td| d| |<|||<| | ||gqpt| |j| S)am Adds pseudo-token disambiguation symbols #1, #2 and so on at the ends of tokens to ensure that all pronunciations are different, and that none is a prefix of another. See also add_lex_disambig.pl from kaldi. Args: lexicon: Lexicon object. Returns: Return Lexicon augmented with subseqence disambiguation symbols. rrPrvr#disambig_subsequence)rVrUrrqtuplecopypoprRr8r_lenrkr+lstriprrr*r4r7)rrrlast_used_disambig_idcount_rnissubseqword_idr6r8first_allowed_disambigfirst_allowed_disambig_id max_disambiglast_used_disambig_id_of token_keycur_disambig_id cur_disambigrrradd_disambig_symbolssV         $rc Cs(td}d}||jv}|r|j|d}t}|}||_|j|dd|j|tj|j ||j|d|dd|rU|}|j|tj|j ||j|d|dd|D]W\} } |} |re| |dkrt | dd D]\} } |}|j| tj| | dkr| ndd|dd|} qmt | d} |j| tj| dkr| d nd| dkr| ndd|ddqW|rVgg}}t |t |jdD]>} |j |j| jt|} |j| }|jd r|| | fq|jd kr|| | fqtd |jd |jd|}|D]\} } |j|tj| | d|ddq|}|D]!\} } |j|tj| | d|dd|j|tj| | d|ddq |j|tj|j ||j|d|dd|rt}|j D] \}} |j|| dqb||_t}|jD] \}} |j|| dq{||_tj|dd|S)z Compiles a Lexicon into a lexicon WFST (L.fst). See also make_lexicon_fst.py from kaldi. Args: lexicon: Lexicon object. Returns: Kaldi-type lexicon WFST. rvrrPrrrrrNrLrr,zUnexpected mark `z` for tokenword ``r~rr)rrVrrrstart set_finalrrrU enumeraterrangerr7r+rrr8r-r]rk RuntimeErrorrrTr input_symbolsrr)rrtrrrrfst start_statetokenword_state_beginrrn cur_staterFtoken_id next_statetokenword_begintokenword_other token_unittokenword_state_maintokenword_state_endisymrrrrrrmake_lexicon_fst_no_silences                rr+rUwith_self_loopscCst|dkr t||}n|dkrt||}n|dkr t|}ntd||rFt}|D] \}}|j||dq1||_ |j |_ |S)aHelper function to build a topology WFST (T.fst). Args: name: Topology name. Choices: default, compact, minimal token2id: Token index. Mapping from token_str to token_id. with_self_loops: Whether to add token-to-epsilon self-loops to the topology. attach_symbol_table: Whether to attach the token names for indices of the returned WFST. Returns: Kaldi-type topology WFST. defaultcompactminimalzUnknown topo name: r~) rbuild_default_topobuild_compact_topobuild_minimal_topo ValueErrorrrrTrrrr)r+rUrrtrrrrFrrr build_topoys    rc Csttd}|d}t}|}||_|j|dd|j|tj |dd|ddg}i}| D]S\}} |dks@|dkrAq4| |rL| | q4|} |j| dd| || <|j|tj | | d| dd|ry|j| tj | dd| dd|j| tj |dd|ddq4t |D];} | dkrt |D]} | dkr| | kr|| } |j| tj | | d| ddq|D]}|j| tj d|d| ddqq|S)z)Build the default (correct) CTC topology.r5rrrrrrO)rrorprrrrrrrrTr\rk StateIterator)rUrr9blank_idrr disambig_idsrnrbrFristateostater disambig_idrrrrs         rc Csttd}|d}t}|}||_|j|dd|j|tj |dd|dd| D]Q\}}|dks<|dkr=q0| |rQ|j|tj d|d|ddq0|}|j|tj ||d|dd|rs|j|tj |dd|dd|j|tj ddd|ddq0|S)zBuild the Compact CTC topology.r5rrrrrrO rrorprrrrrrrrTr\) rUrr9rrrrbrFrrrrrsv       rc Csttd}|d}t}|}||_|j|dd|j|tj |dd|dd| D]/\}}|dks<|dkr=q0| |rQ|j|tj d|d|ddq0|j|tj ||d|ddq0|S)zBuild the Minimal CTC topology.r5rrrrrrOr)rUr9rrrrbrFrrrr>sN     rrrrkaldi topology_namewrite_tlg_pathopen_vocabularyopen_vocabulary_weightstarget)ruzk2.Fsacsttdt|}|r t||j|d|dd}nd}tdddt|j  d D} t |j| d |d } t | } t| } tj| d d tdt| |} t| t| tj| dd tdt|| j}tj|d d t|| }|dkr|rtd|d||||fS|dkrtdddl}ddlm}dd| jDd}dd| jDd| j| jj}| j|jdksJd fdd| jD}t|j }d|_!d|_ |"d~|j#j$|j%d ddd}||j&|j&k<d|j'|j'j(|k<d|j)d <|*|+|,|}|j&|j&dkd|j&|j&dk<d|j)d <|j-.||_/|j-.||_0|*|}|rltd!|d|1|2|Wd||fSWd||fS1szwY||fSt3d"|d#)$a Builds a decoding WFST (TLG.fst or TLG.pt). See also mkgraph.sh from kaldi. Args: tokenizer: NeMo SentencePiece tokenizer. lm_path: Path to the ARPA LM file. topology_name: Topology name. Choices: default, compact, minimal. write_tlg_path: Where to buffer the TLG. open_vocabulary: Whether to build a decoding WFST suitable for the open vocabulary decoding. open_vocabulary_weights: Pair of weights (oov_word_weight, token_unigram_weight). target: What type to build the WFST for. Choices: kaldi, k2. Returns: A pair of kaldi- or k2-type decoding WFST and its id of the tokenword disambiguation token. zCompiling G.fst ...rrP)rrrrrLzBuilding L.fst ...cS*i|]}t|dd|ddqS rPrrqrr=rrrrr@s*z"mkgraph_ctc_ov.. T)rrrrzBuilding LG.fst ...rzBuilding TLG.fst ...r!zBuffering TLG.fst into z ...k2zConverting TLG.fst to k2 ...N)r-cSg|] \}}|jdkr|qS)rr-r=rFrrrrrIrKz"mkgraph_ctc_ov..cSr.)rr/r0rrrrIrKrcs:g|]\}}d|krkrnn |d|dqS)r rPrr<first_token_disambig_idrrrIs:Fshow_weight_one)acceptor _propertieszBuffering TLG.pt into zUnsupported target: `r)4rrinforrrrrr2rrrrrrrrcomposedeterminize_starminimize_encodedrrUwritetorchnemo.core.utils.k2_guardr-r8rTrVr+r7r-rrinference_modeFsa from_openfstto_strlabels aux_labelsrS__dict__arc_sortconnectremove_epsilonrfrom_str labels_symaux_labels_symsaveas_dictr)rrsr"r#r$r%r&rrr7rlexicon_disambigLLGTTLGr=r-rword_disambig_idrrrr2rmkgraph_ctc_ovms'            #          rTc@s@eZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdS) KaldiFstMaskill ii@iii@illlN)r.r/r0AcceptorError TopSortedAcyclic IlabelSorted OlabelSortedIlabelDeterministicOlabelDeterministic HasEpsilons HasIEpsilons Accessible CoaccessibleWeightedrrrrrUsrUc@sfeZdZUeed<eed<eed<eed<eed<eed<eed<eed<eed <eed <eed <d S) LatticePropertiesrWValidNonemptyrYrZ ArcSorted Deterministic EpsilonFreeInputEpsilonFree ConnectedrcN)r.r/r0boolr3rrrrrds  rdc @seZdZdZdefddZedddZe dd ee e e fd ee e e fd e de dfddZ edeedefddZeddZedefddZedeeee ffddZedeeefddZd S)AbstractLatticez/A lattice wrapper with high-level capabilities.latticecCs||_d|_dSre)_latticer7)rHrnrrrrcs zAbstractLattice.__init__rd torch.TensorcCdS)URepresents the lattice as a tensor. Returns: torch.Tensor NrrGrrr as_tensorszAbstractLattice.as_tensorN?filenametitlezoomzgraphviz.DigraphzIPython.display.HTMLcCrq)Render FSA as an image via graphviz, and return the Digraph object; and optionally save to file filename. filename must have a suffix that graphviz understands, such as pdf, svg or png. Note: You need to install graphviz to use this function:: ./scripts/installers/install_graphviz.sh Args: filename: Filename to (optionally) save to, e.g. ‘foo.png’, ‘foo.svg’, ‘foo.png’. title: Title to be displayed in image, e.g. ‘A simple lattice example’. zoom: Zoom-in lattice in IPython notebook (needed for large lattices). Returns: graphviz.Digraph or IPython.display.HTML Nr)rHrurvrwrrrdrawszAbstractLattice.drawreference_sequencecCrq)Get the edit distance from a reference sequence to the lattice. Args: reference_sequence: List of word- or token-ids. Returns: Number of edits. Nr)rHr{rrr edit_distance*szAbstractLattice.edit_distancecCs d|_|jSre)r7rorGrrrrn6szAbstractLattice.latticecCdSrerrGrrr properties;zAbstractLattice.propertiescCr~rerrGrrr symbol_table?rzAbstractLattice.symbol_tablecCr~rerrGrrrauxiliary_tablesCrz AbstractLattice.auxiliary_tablesrdrpNNrt)r.r/r0r1r rcrrsrrr r2floatrzr rqr}rrrnrrdrr rrrrrrrrms4   rmc seZdZdZ  ddddeeeefdeeeefffdd Z e d e fd d Z e d eeeeffd d Z e d eeefddZdddZdeed efddZ ddeeeefdeeeefded edfddZZS) KaldiWordLatticez5A Kaldi lattice wrapper with high-level capabilities.Nrnkaldifst.Latticerrcstt|tjstdt|dt|dd}d|_|dur(||_n |j j dur5||j j |_d|_ |durkt | t |}}d|vra|j jdura|d|||j jtd|||_ dS|j jdurtdd||j j|_ dSdS)NzWrong lattice type: `rcSsddt|dDS)NcSr'r(r*r+rrrr@Wsz?KaldiWordLattice.__init__....r,)r2rr)symbolsrrrrWsz+KaldiWordLattice.__init__..rKaldiAuxiliaryTables)rrQrLatticertypesuperrc _symbol_tableror_auxiliary_tablesrRrrSrrkr)rHrnrrkaldi_symbols2dict attributesrS __class__rrrcKs,       zKaldiWordLattice.__init__rdc Cs|jdur|jtjjdtjjk}|jtjjdtjjk}|jjdk}|jtjjdtjjk}|jtj jdtj jk}|jtj jdtj jkoX|jtj jdtj jk}|jtj jdtj jkor|jtj jdtj jk}|jtjjdtjjk}|jtjjdtjjk} |jtjjdtjjko|jtjjdtjjk} |jtjjdtjjk} t||||||||| | | d |_|jS)NTr) rWrerfrYrZrgrhrirjrkrc)r7rorrUrWrrX num_statesrYrZr[r\r]r^r_r`rarbrcrd) rHr6validnonempty top_sortedacyclic arc_sorted deterministic epsilon_freeinput_epsilon_free connectedweightedrrrrosL   zKaldiWordLattice.propertiescC|jSre)rrGrrrrzKaldiWordLattice.symbol_tablecCrre)rrGrrrrrz!KaldiWordLattice.auxiliary_tablesrpcCstd)rrz+Tensor representation is not supported yet.)NotImplementedErrorrGrrrrsszKaldiWordLattice.as_tensorr{cCst|jjs tdt|stdtt |}t|j }t |t ||}t t|\}}}}|s=tdt|jS)r|zPLattice contains input epsilons. Edit distance calculations may not be accurate.z8reference_sequence contains zeros, which is not allowed.zRSomething went wrong while calculating edit_distance. Please check input manually.)rrrjrwarningallrlevenshtein_graph_kaldirmake_linear_acceptorroinvertr9get_linear_symbol_sequence shortest_pathrroundr)rHr{refhypali_fst succeededr total_weightrrrr}s      zKaldiWordLattice.edit_distancertrurvrwrxcCsRttd\}}|jr"t}|jD] \}}|j||dq|jrEt|jdrE|jj rEt}|jj D] \}}|j||dq9tj |j dd||dd} t | } t| } | d| dd d d d d ddd} |durx|| d<t j| d} | j| 7_|rtj|\}}|dks|ddkrtd|dt}| jd||dddd}t||Wdn1swYtr'ddl}ddlm}t}| jd|ddd}t ||\}\}}Wdn1swY|!|"d|d|dt#||d d!WdS1s"wY| S)"ryNNr~rFT)r6portraitisymbolsosymbolsr5rrLLRz8.5,111Portraitz0.4z0.25z0.0)rankdirsizecenter orientationranksepnodesepmarginNr) graph_attrr,.z@Filename needs to have a suffix like .png, .pdf, .svg, or .gv: `rtemprP)ru directoryformatcleanup)HTMLsvgignorez)$rr"rrrrTrrhasattrrrzror#Sourcer2 splitlinesrDigraphbodyrrsplitextrrrrendershutilmove _is_notebookwarningsIPython.displayr_svg_srcdoc_resizecatch_warnings simplefilterr)rHrurvrwrrrFrrfst_dotsource source_linesrdigraphr extensiontmp_dirtemp_fnrrrwidthheightrrrrzsr            "zKaldiWordLattice.drawrrr)r.r/r0r1rr rqr2r rcrrrdrrrrrsr r}rr rrz __classcell__rrrrrHs:$+ rcCs.z tjj}|dks WdStyYdSw)NZMQInteractiveShellShellTTerminalInteractiveShellF) get_ipythonrr. NameError)shellrrrrs  rrurwcCst|dddD}|}|ds|}|drtd|\}}t|t|}}dt||dt||d|||ffWdS1sNwYdS) Nrtr{r|zJt|dksFJ| d} t}t||||dd|| <| } | |_ |d7}q| dvr| dkrd d | dd D\} }}| d d }d d |dd D\}}nt d| ddd | D\} }}dg}d\}}||krt ||dD]}| q|d}tj||tj||d|d}|j| |d|||d dkrdd |d tDngfq| dkrt| d} | d d }dd |dd D\}}|j| tj||ddqtd|dWd|S1s0wY|S)aHelper function to load riva-decoder recognition lattices. Args: lat_filename: Path to the riva-decoder recognition lattice file. id2word: Word index. Mapping from word_id to word_str. id2token: Token index. Mapping from token_id to token_str. Returns: Dictionary with lattice names and corresponding lattices in KaldiWordLattice format. NrrrP)token_seq_listr)rnrr)rrVrVcSg|]}t|qSrrqrErrrrIrz%load_word_lattice..rL,cSrrrrErrrrIrz/An arc without weight is detected for lattice `zY`. Weights and token sequences will be set trivially.cSrrrrErrrrIrr,)rr) graph_cost acoustic_costrrcSrrrrErrrrIrrcSrrrrErrrrIrrzBroken line: `r)rr readlinesrrrrrrrrrrr LatticeArc LatticeWeightrrkrrqrr)rr7r8 lattice_dictrn max_staterrr line_itemsline_lenlat_idrrrrtrunkrrrFarkrrrload_word_latticets~        4   88r)T)rrr)TT)rNFr r!)rr)Grrorabcrrr collectionsrr dataclassesrenumrpathlibr typingr r r r rrr nemo.utilsrrrrrrModuleNotFoundErrorAttributeErrorr#r r(r%rr!r&rrr"r$r'r)r*r4r2rlrrrqrrprrrrrrrrTrUrdrmrrrrrrrrrs2   $ W- R  ZN   * XC2   tIW.   @