o %ݫiA@s&dZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z m Z ddlZddlZddlmZddlmZeeZGdddZdd d Zdd d ZejGdddZejGdddeZejGdddZGdddejjZ Gddde Z!Gddde Z"GdddZ#dS)zDecoders and output normalization for CTC. Authors * Mirco Ravanelli 2020 * Aku Rouhe 2020 * Sung-Lin Yeh 2020 * Adel Moumen 2023, 2024 N)groupby)AnyDictListOptionalTupleUnion)length_to_mask) get_loggerc@s4eZdZdZd ddZed ddZdd ZdS) CTCPrefixScoreaThis class implements the CTC prefix score of Algorithm 2 in reference: https://www.merl.com/publications/docs/TR2017-190.pdf. Official implementation: https://github.com/espnet/espnet/blob/master/espnet/nets/ctc_prefix_score.py Arguments --------- x : torch.Tensor The encoder states. enc_lens : torch.Tensor The actual length of each enc_states sequence. blank_index : int The index of the blank token. eos_index : int The index of the end-of-sequence (eos) token. ctc_window_size: int Compute the ctc scores over the time frames using windowing based on attention peaks. If 0, no windowing applied. rc Cs.||_||_|d|_|d|_|d|_|j|_d|_|d|_||_ d|_ dt |}| d dd|dd}|||j|dddddf|dddddfd|dddddf<|dd}|dddd|jf d dd|j}t||g|_tj|j|jd|_dS)Nrg@xdevice) blank_index eos_indexsize batch_size max_enc_len vocab_sizer minus_inflast_frame_indexctc_window_size prefix_lengthr unsqueezeexpandeq masked_fill_ transposetorchstackxarange batch_index) selfr"enc_lensrrrmaskxnbxbr*L/home/ubuntu/.local/lib/python3.10/site-packages/speechbrain/decoders/ctc.py__init__.s*      B  zCTCPrefixScore.__init__Nc"Cs|d}||j}|}|jd7_|dur|jn|d|_|durctj|jd|j|f|j|j d}t |j ddddd|j fd d|dddf<|dd|}tj||jfd|j d} n|\}} |dur|j|j} tj||jfdtj|j d} tj||j d d} tj|j|j d| | |f<|| dd|ddd} t|j dd|j|jd| dd||j}nd} |j d ddd|ddd||j}tj|jd||jf|j|j d}||j|jdkr|d |d <t|d}| ddd|j}|dur5t|D] }| |||f}|dkr2|ddd|f|dd||f<qnt|D]}|ddd|f|dd|||f<q9|jdksZ|durdtd|j}|j}n0tj|dd \}}t||j}t||j}ttd|jt|}t|jt|}t||D]7}||ddf}||ddf}t|||d||gdd||j}t|d|dd|f||<q||ddf d}tj|d d|ddfdd |d}|dur)tj||jf|j|j d} tjtj||||fdd dd }!t|D] }|!|| |||f<qntjtj||||fdd dd } t|D]}||j|||f| ||jf<q?|j|j krd|j| dd|j f<| | || | ffS) aThis method if one step of forwarding operation for the prefix ctc scorer. Arguments --------- inp_tokens : torch.Tensor The last chars of prefix label sequences g, where h = g + c. states : tuple Previous ctc states. candidates : torch.Tensor (batch_size * beam_size, ctc_beam_size), The topk candidates for rescoring. If given, performing partial ctc scoring. attn : torch.Tensor (batch_size * beam_size, max_enc_len), The attention weights. Returns ------- new_psi : torch.Tensor (r, psi, scoring_table) : tuple rr Nr rr)dtyper)rrdim) rrrrnum_candidatesr fullrrrcumsumr"rrviewr$longr#repeat index_selectfill_ logsumexprangermaxitemminintr!catrr)"r% inp_tokensstates candidatesattnn_bh beam_size last_charr_prevpsi_prev cand_offset scoring_table col_index scoring_index x_inflaterr_sumphiiposstartend_ attn_peak max_frame min_frametrnb_prevrb_prevr_psi_initphixpsipsi_r*r*r+ forward_stepNs             (  $,   zCTCPrefixScore.forward_stepc Cs|\}}}|d}|j|}|j|}||d||j|} tj|dd| d}|dd d|j||j}tj ||jdd|d||} |durr||jd} || | f} d| | dk<| | |j } tj|dd||j d| d}|dd|}||fS) a~This method permutes the CTC model memory to synchronize the memory index with the current output. Arguments --------- memory : No limit The memory variable to be permuted. index : torch.Tensor The index of the previous path. Return ------ The variable of the memory being permuted. r r r)r1indexfloor) rounding_modeNr) rrr$r expand_asrr5r r8r7divr2) r%memoryrcrOr`rKrFrE beam_offset cand_index hyp_indexselected_vocab score_indexr*r*r+ permute_mems6         zCTCPrefixScore.permute_memrNN) __name__ __module__ __qualname____doc__r,r no_gradrbrnr*r*r*r+r s   %r r cs>t|trddt|D}ttfdd|}|Std)acApply CTC output merge and filter rules. Removes the blank symbol and output repetitions. Arguments --------- string_pred : list A list containing the output strings/ints predicted by the CTC system. blank_id : int, string The id of the blank. Returns ------- list The output predicted by CTC without the blank symbol and the repetitions. Example ------- >>> string_pred = ['a','a','blank','b','b','blank','c'] >>> string_out = filter_ctc_output(string_pred, blank_id='blank') >>> print(string_out) ['a', 'b', 'c'] cSsg|]}|dqSror*).0rRr*r*r+ Fsz%filter_ctc_output..cs|kSNr*)elemblank_idr*r+Isz#filter_ctc_output..z+filter_ctc_out can only filter python lists) isinstancelistrfilter ValueError) string_predr{ string_outr*rzr+filter_ctc_output*s rc Cst|tr|dkr|jd|}|jd}g}t||D](\}}tt||}tj|dd|dd\}} t| |d} | | q|S)aYGreedy decode a batch of probabilities and apply CTC rules. Arguments --------- probabilities : torch.tensor Output probabilities (or log-probabilities) from the network with shape [batch, lengths, probabilities] seq_lens : torch.tensor Relative true sequence lengths (to deal with padded inputs), the longest sequence has length 1.0, others a value between zero and one shape [batch, lengths]. blank_id : int, string The blank symbol/index. Default: -1. If a negative number is given, it is assumed to mean counting down from the maximum possible index, so that -1 refers to the maximum possible index. Returns ------- list Outputs as Python list of lists, with "ragged" dimensions; padding has been removed. Example ------- >>> import torch >>> probs = torch.tensor([[[0.3, 0.7], [0.0, 0.0]], ... [[0.2, 0.8], [0.9, 0.1]]]) >>> lens = torch.tensor([0.51, 1.0]) >>> blank_id = 0 >>> ctc_greedy_decode(probs, lens, blank_id) [[1], [1]] rr r r0rz) r}r?shapezipr roundr<narrowrtolistappend) probabilitiesseq_lensr{ batch_max_len batch_outputsseqseq_len actual_sizescores predictionsoutr*r*r+ctc_greedy_decodeOs!  rc@seZdZUdZeed<eed<eed<eed<eeed<eeed<ee eefed<e eefed <e j Z e ed <e j Ze ed <e j Ze ed <e j Ze ed <e j Ze ed<e j Ze ed<e j Ze ed<edddZdddZdS)CTCBeamalThis class handle the CTC beam information during decoding. Arguments --------- text : str The current text of the beam. full_text : str The full text of the beam. next_word : str The next word to be added to the beam. partial_word : str The partial word being added to the beam. last_token : str, optional The last token of the beam. last_token_index : int, optional The index of the last token of the beam. text_frames : List[Tuple[int, int]] The start and end frame of the text. partial_frames : Tuple[int, int] The start and end frame of the partial word. p : float The probability of the beam. p_b : float The probability of the beam ending in a blank. p_nb : float The probability of the beam not ending in a blank. n_p_b : float The previous probability of the beam ending in a blank. n_p_nb : float The previous probability of the beam not ending in a blank. score : float The score of the beam (LM + CTC) score_ctc : float The CTC score computed. Example ------- >>> beam = CTCBeam( ... text="", ... full_text="", ... next_word="", ... partial_word="", ... last_token=None, ... last_token_index=None, ... text_frames=[(0, 0)], ... partial_frames=(0, 0), ... p=-math.inf, ... p_b=-math.inf, ... p_nb=-math.inf, ... n_p_b=-math.inf, ... n_p_nb=-math.inf, ... score=-math.inf, ... score_ctc=-math.inf, ... ) text full_text next_word partial_word last_tokenlast_token_index text_framespartial_framespp_bp_nbn_p_bn_p_nbscore score_ctclm_beam LMCTCBeamreturncCsDt|j|j|j|j|j|j|j|j|j |j |j |j |j |j|jdS)zCreate a CTCBeam from a LMCTCBeam Arguments --------- lm_beam : LMCTCBeam The LMCTCBeam to convert. Returns ------- CTCBeam The CTCBeam converted. )rrrrrrrrrrrrrrr)rrrrrrrrrrrrrrrr)r%rr*r*r+ from_lm_beams"zCTCBeam.from_lm_beamNcCs@|j|j|_|_tj |_|_t|j|j|_|j|_ dS)zUpdate the beam probabilities.N) rrrrmathinfnp logaddexprrr%r*r*r+steps z CTCBeam.step)rrrr)rN)rqrrrsrtstr__annotations__rr?rrrrrfloatrrrrrr classmethodrrr*r*r*r+r|s( 8   rc@s"eZdZUdZej Zeed<dS)rzThis class handle the LM scores during decoding. Arguments --------- lm_score: float The LM score of the beam. **kwargs See CTCBeam for the other arguments. lm_scoreN) rqrrrsrtrrrrrr*r*r*r+rs  rc@s>eZdZUdZeed<ded<eed<eed<dZeed<dS) CTCHypothesisadThis class is a data handler over the generated hypotheses. This class is the default output of the CTC beam searchers. It can be re-used for other decoders if using the beam searchers in an online fashion. Arguments --------- text : str The text of the hypothesis. last_lm_state : None The last LM state of the hypothesis. score : float The score of the hypothesis. lm_score : float The LM score of the hypothesis. text_frames : List[Tuple[str, Tuple[int, int]]], optional The list of the text and the corresponding frames. rN last_lm_staterrr) rqrrrsrtrrrrr~r*r*r*r+rs rc!s$eZdZdZ           dDd edeedededefdedeefdededede dededede dedefdedef fdd Z dEd e j d!ee d"ed#ed$ef d%d&Zd'ed(efd)d*Zd+ed,ed(efd-d.Zd!ee d(ee fd/d0Zd!ee d(ee fd1d2Zd!ee d3ed(ee fd4d5Z 6 6dFd!ee d"ed#ed(ee fd7d8Z  dGd e j d9ee j d:ed(eeefd;d<Z  dGd e j d9ee j d:ed(eeefd=d>Z 6 6dFd e j d"ed#ed!ee d$ed(ee f d?d@Z dHd e j dAed:eed(eefdBdCZZS)ICTCBaseSearcherur CTCBaseSearcher class to be inherited by other CTC beam searchers. This class provides the basic functionalities for CTC beam search decoding. The space_token is required with a non-sentencepiece vocabulary list if your transcription is expecting to contain spaces. Arguments --------- blank_index : int The index of the blank token. vocab_list : list The list of the vocabulary tokens. space_token : int, optional The index of the space token. (default: -1) kenlm_model_path : str, optional The path to the kenlm model. Use .bin for a faster loading. If None, no language model will be used. (default: None) unigrams : list, optional The list of known word unigrams. (default: None) alpha : float Weight for language model during shallow fusion. (default: 0.5) beta : float Weight for length score adjustment of during scoring. (default: 1.5) unk_score_offset : float Amount of log score offset for unknown tokens. (default: -10.0) score_boundary : bool Whether to have kenlm respect boundaries when scoring. (default: True) beam_size : int, optional The width of the beam. (default: 100) beam_prune_logp : float, optional The pruning threshold for the beam. (default: -10.0) token_prune_min_logp : float, optional The pruning threshold for the tokens. (default: -5.0) prune_history : bool, optional Whether to prune the history. (default: True) Note: when using topk > 1, this should be set to False as it is pruning a lot of beams. blank_skip_threshold : float, optional Skip frames if log_prob(blank) > log(blank_skip_threshold), to speed up decoding. Note: This is only used when using the CUDA decoder, and it might worsen the WER/CER results. Use it at your own risk. (default: 1.0) topk : int, optional The number of top hypotheses to return. (default: 1) spm_token: str, optional The sentencepiece token. (default: "▁") Example ------- >>> blank_index = 0 >>> vocab_list = ['blank', 'a', 'b', 'c', ' '] >>> space_token = ' ' >>> kenlm_model_path = None >>> unigrams = None >>> beam_size = 100 >>> beam_prune_logp = -10.0 >>> token_prune_min_logp = -5.0 >>> prune_history = True >>> blank_skip_threshold = 1.0 >>> topk = 1 >>> searcher = CTCBaseSearcher( ... blank_index=blank_index, ... vocab_list=vocab_list, ... space_token=space_token, ... kenlm_model_path=kenlm_model_path, ... unigrams=unigrams, ... beam_size=beam_size, ... beam_prune_logp=beam_prune_logp, ... token_prune_min_logp=token_prune_min_logp, ... prune_history=prune_history, ... blank_skip_threshold=blank_skip_threshold, ... topk=topk, ... )  N??$Td?r ▁r vocab_list space_tokenkenlm_model_pathunigramsalphabetaunk_score_offsetscore_boundaryrFbeam_prune_logptoken_prune_min_logp prune_historyblank_skip_thresholdtopk spm_tokencst|_|_|_|_|_|_|_|_ | _ | _ | _ | _ | _t|_|_|_tfdd|D_jsoz||_Wntydtd|dd_Ynwtdjdd_|durzddl}dd lm}m }Wn t!yt!d w|"|_|dur|#d rtd |dur|dur|#d r||}ntd jdur|j|jjj j d_$dSd_$dS)Ncsg|] }t|jqSr*)r startswithr)rvsrr*r+rwsz,CTCBaseSearcher.__init__..z space_token `z` not found in the vocabulary.Using value -1 as `space_index`.Note: If your transcription is not expected to contain spaces, you can ignore this warning.r zFound `space_token` at index .r) LanguageModelload_unigram_set_from_arpazwkenlm python bindings are not installed. To install it use: pip install https://github.com/kpu/kenlm/archive/master.zipz.arpazJUsing arpa instead of binary LM file, decoder instantiation might be slow.zUnigrams not provided and cannot be automatically determined from LM file (only arpa format). Decoding accuracy might be reduced.) kenlm_modelrrrrr)%superr,rrrrrrrrrrFrrrrlogrrranyis_spmrc space_indexrloggerwarninginforkenlm#speechbrain.decoders.language_modelrr ImportErrorModelendswithlm)r%rrrrrrrrrrFrrrrrrrrr __class__rr+r,hsz           zCTCBaseSearcher.__init__r log_probsbeamscached_lm_scorescached_p_lm_scoresprocessed_framescCst)aPerform a single step of decoding. Arguments --------- log_probs : torch.Tensor The log probabilities of the CTC output. beams : list The list of the beams. cached_lm_scores : dict The cached language model scores. cached_p_lm_scores : dict The cached prefix language model scores. processed_frames : int, default: 0 The start frame of the current decoding step. )NotImplementedError)r%rrrrrr*r*r+partial_decodingsz CTCBaseSearcher.partial_decodingrrcCsd|S)zEfficiently normalize whitespace. Arguments --------- text : str The text to normalize. Returns ------- str The normalized text. r)joinsplit)r%rr*r*r+normalize_whitespaces z$CTCBaseSearcher.normalize_whitespacetoken_1token_2cCs8t|dkr |}|St|dkr|}|S|d|}|S)aOMerge two tokens, and avoid empty ones. Taken from: https://github.com/kensho-technologies/pyctcdecode Arguments --------- token_1 : str The first token. token_2 : str The second token. Returns ------- str The merged token. rr)len)r%rrrr*r*r+ merge_tokenss   zCTCBaseSearcher.merge_tokenscCsli}|D]+}||j|j}||j|jf}||vr|||<qtj|t||j |j d||<qt | S)a#Merge beams with the same text. Taken from: https://github.com/kensho-technologies/pyctcdecode Arguments --------- beams : list The list of the beams. Returns ------- list The list of CTCBeam merged. r) rrrrr dataclassesreplacerrrr~values)r%r beam_dictbeamnew_texthash_idxr*r*r+ merge_beams s  zCTCBaseSearcher.merge_beamscCstj|j|dddS)zSort beams by lm_score. Arguments --------- beams : list The list of CTCBeam. Returns ------- list The list of CTCBeam sorted. cSs|jSrxr)r"r*r*r+r|7sz,CTCBaseSearcher.sort_beams..)key)heapqnlargestrF)r%rr*r*r+ sort_beams*s zCTCBaseSearcher.sort_beamslm_ordercCsjtd|d}t}g}|D]$}t|j| d|j|jf}||vr2|t || |q|S)a5Filter out beams that are the same over max_ngram history. Since n-gram language models have a finite history when scoring a new token, we can use that fact to prune beams that only differ early on (more than n tokens in the past) and keep only the higher scoring ones. Note that this helps speed up the decoding process but comes at the cost of some amount of beam diversity. If more than the top beam is used in the output it should potentially be disabled. Taken from: https://github.com/kensho-technologies/pyctcdecode Arguments --------- beams : list The list of the beams. lm_order : int The order of the language model. Returns ------- list The list of CTCBeam. r N) r<settuplerrrrrrradd)r%rr min_n_history seen_hashesfiltered_beamsrrr*r*r+_prune_history9s zCTCBaseSearcher._prune_historyFc s|s|r4g}|D]%}|jdkr|jn|j|jg}|t|j|j|jddd|d|jd q|}nt |} |||} t dd| Dfdd| D}  | } | S)aeFinalize the decoding process by adding and scoring the last partial word. Arguments --------- beams : list The list of CTCBeam. cached_lm_scores : dict The cached language model scores. cached_p_lm_scores : dict The cached prefix language model scores. force_next_word : bool, default: False Whether to force the next word. is_end : bool, default: False Whether the end of the sequence has been reached. Returns ------- list The list of the CTCBeam. Nr r rrrrrrrrrcSg|]}|jqSr*rrvbr*r*r+rwz5CTCBaseSearcher.finalize_decoding..c g|] }|jjkr|qSr*rrr  max_scorer%r*r+rw ) rrrrrrrrrr~ get_lm_beamsr<r) r%rrrforce_next_wordis_end new_beamsrnew_token_times scored_beams sorted_beamsr*rr+finalize_decodingcs>     z!CTCBaseSearcher.finalize_decodingwav_lenslm_start_statecs|dtjkrtd|ddtjd|dur1|d|}|t}n |dg|d}|}fdd t ||D}|S) a}Decodes the input log probabilities of the CTC output. It automatically converts the SpeechBrain's relative length of the wav input to the absolute length. Make sure that the input are in the log domain. The decoder will fail to decode logits or probabilities. The input should be the log probabilities of the CTC output. Arguments --------- log_probs : torch.Tensor The log probabilities of the CTC output. The expected shape is [batch_size, seq_length, vocab_size]. wav_lens : torch.Tensor, optional (default: None) The SpeechBrain's relative length of the wav input. lm_start_state : Any, optional (default: None) The start state of the language model. Returns ------- list of list The list of topk list of CTCHypothesis. rz,Vocab size mismatch: log_probs vocab dim is z while vocab_list is zQ. During decoding, going to truncate the log_probs vocab dim to match vocab_list.Nr rcsg|] \}}||qSr*)decode_log_probs)rvlog_probwav_lenrr%r*r+rws z0CTCBaseSearcher.decode_beams..) rrrwarningswarncpunumpyastyper?r)r%rrrhypsr*r r+ decode_beamss  zCTCBaseSearcher.decode_beamscCs||||S)aDecodes the log probabilities of the CTC output. It automatically converts the SpeechBrain's relative length of the wav input to the absolute length. Each tensors is converted to numpy and CPU as it is faster and consumes less memory. Arguments --------- log_probs : torch.Tensor The log probabilities of the CTC output. The expected shape is [batch_size, seq_length, vocab_size]. wav_lens : torch.Tensor, optional (default: None) The SpeechBrain's relative length of the wav input. lm_start_state : Any, optional (default: None) The start state of the language model. Returns ------- list of list The list of topk list of CTCHypothesis. r')r%rrrr*r*r+__call__szCTCBaseSearcher.__call__c Cs,|j|||||d}|j|||||d}|S)aPerform a single step of decoding. Arguments --------- log_probs : torch.Tensor The log probabilities of the CTC output. cached_lm_scores : dict The cached language model scores. cached_p_lm_scores : dict The cached prefix language model scores. beams : list The list of the beams. processed_frames : int The start frame of the current decoding step. force_next_word : bool, optional (default: False) Whether to force the next word. is_end : bool, optional (default: False) Whether the end of the sequence has been reached. Returns ------- list The list of CTCBeam. )rrr)rr) r%rrrrrrr trimmed_beamsr*r*r+partial_decode_beamss"z$CTCBaseSearcher.partial_decode_beamsrc sj}|dur in|dur|}n|}dd|fii}tddddddgddddd g}||||}j||ddd}fd d |Ddj} | S) aDecodes the log probabilities of the CTC output. Arguments --------- log_probs : torch.Tensor The log probabilities of the CTC output. The expected shape is [seq_length, vocab_size]. wav_len : int The length of the wav input. lm_start_state : Any, optional (default: None) The start state of the language model. Returns ------- list The topk list of CTCHypothesis. N)rFr-rr rrrrrrrrrrrTr*c s\g|]*}t|j|jdfvr|jdfdndtt|j|j|j|jdqS)Tr N)rrrrr) rrrr~rrrrr)rvrrr%r*r+rwos z4CTCBaseSearcher.decode_log_probs..)rget_start_staterrrr) r%rrrlanguage_model start_staterrr+ output_beamsr*r.r+r-sL    z CTCBaseSearcher.decode_log_probs)rNNrrrTrrrTrr rro)FFrprx)rqrrrsrtr?rrrrboolr,r Tensorrdictrrrrrrrrrrr'r)r,r __classcell__r*r*rr+rsP       j  / G  7  % 8rc@sjeZdZdZ ddeedededeefddZ dd e j d e deeded ed e deefddZ dS)CTCBeamSearcherasCTC Beam Search is a Beam Search for CTC which does not keep track of the blank and non-blank probabilities. Each new token probability is added to the general score, and each beams that share the same text are merged together. The implementation supports n-gram scoring on words and SentencePiece tokens. The input is expected to be a log-probabilities tensor of shape [batch, time, vocab_size]. The main advantage of this CTCBeamSearcher over the CTCPrefixBeamSearcher is that it is relatively faster, and obtains slightly better results. However, the implementation is based on the one from the PyCTCDecode toolkit, adapted for the SpeechBrain's needs and does not follow a specific paper. We do recommend to use the CTCPrefixBeamSearcher if you want to cite the appropriate paper for the decoding method. Several heuristics are implemented to speed up the decoding process: - pruning of the beam : the beams are pruned if their score is lower than the best beam score minus the beam_prune_logp - pruning of the tokens : the tokens are pruned if their score is lower than the token_prune_min_logp - pruning of the history : the beams are pruned if they are the same over max_ngram history - skipping of the blank : the frame is skipped if the blank probability is higher than the blank_skip_threshold Note: if the Acoustic Model is not trained, the Beam Search will take a lot of time. We do recommend to use Greedy Search during validation until the model is fully trained and ready to be evaluated on test sets. Arguments --------- see CTCBaseSearcher, arguments are directly passed. Example ------- >>> import torch >>> from speechbrain.decoders import CTCBeamSearcher >>> probs = torch.tensor([[[0.2, 0.0, 0.8], ... [0.4, 0.0, 0.6]]]) >>> log_probs = torch.log(probs) >>> lens = torch.tensor([1.0]) >>> blank_index = 2 >>> vocab_list = ['a', 'b', '-'] >>> searcher = CTCBeamSearcher(blank_index=blank_index, vocab_list=vocab_list) >>> hyps = searcher(probs, lens) Frrcached_partial_token_scoresrcCsD|jdur/g}|D]#}||j|j}|t||jd|j|j|j|j |j |j |j d q |Sg}|D]l}||j|j}||f}||vrd||jdf\} } |jj | |j|d\} } | | } | | f||<||\}}|j}t |dkr||vr|j |||<|||7}|t||jd||j|j|j |j |j |j |d q3|S)Score the beams with the language model if not None, and return the new beams. This function is modified and adapted from https://github.com/kensho-technologies/pyctcdecode Arguments --------- beams : list The list of the beams. cached_lm_scores : dict The cached language model scores. cached_partial_token_scores : dict The cached partial token scores. is_eos : bool (default: False) Whether the end of the sequence has been reached. Returns ------- new_beams : list The list of the new beams. Nr) rrrrrrrrrrF is_last_wordr)rrrrrrrrrrrrrscore_partial_tokenr%rrr8is_eosrrr cache_keyprev_raw_lm_scorer1r end_state raw_lm_scorerrV word_partr*r*r+rsl       zCTCBeamSearcher.get_lm_beamsrrrrrcs|d|}t||dD]_\}}|jjkrq |} tt|jkd| hB} g} | ttt j @} | D]} || } j | }|D]}| jksU|j |kr| jkr`|j d}n|d}| jkrl|j n|j d|f}| t|j|j|j|j|| |j||j| d qIjr|ddjkr|dd}|jdkr|jn|j|j g}| t|j|j|j||| |||df|j| d qIjs| jkr|jdkr|jn|j|j g}| t|j|j|jd|| |d|j| d qI|j ddkr||dfn|j d|df}| t|j|j|j|j||| |j||j| d qIq<| } | ||}tdd |Dfd d |D}|}jrejdurYdnjj}j||d }q d d |D}q |S) (Perform CTC Prefix Beam Search decoding. If self.lm is not None, the language model scores are computed and added to the CTC scores. Arguments --------- log_probs : torch.Tensor The log probabilities of the CTC input. Shape: (seq_length, vocab_size) wav_len : int The length of the input sequence. beams : list The list of CTCBeam objects. cached_lm_scores : dict The cached language model scores. cached_p_lm_scores : dict The cached prefix language model scores. processed_frames : int The start frame of the current decoding step. (default: 0) Returns ------- beams : list The list of CTCBeam objects. NrTrr r rrcSr r*rr r*r*r+rwr z4CTCBeamSearcher.partial_decoding..crr*rr rr*r+rwrrcSg|]}t|qSr*rrr r*r*r+rw) enumeraterrargmaxrrwhererr;rrrrrrrrrrrrrrrrrr<rrrorderr)r%rrrrrr frame_index logit_col max_indextokens_index_listr token_indexp_tokentokenr new_end_framenew_part_frames clean_tokennew_frame_listrr+rr*rr+rs #                 g  z CTCBeamSearcher.partial_decodingNFro) rqrrrsrtrrr5rrr r4r?rr*r*r*r+r7s83 fr7c@seZdZdZ ddeedededeefddZd e d e d e d e deed e dedefddZ dde jde deededede deefddZdS)CTCPrefixBeamSearcheraBCTC Prefix Beam Search is based on the paper `First-Pass Large Vocabulary Continuous Speech Recognition using Bi-Directional Recurrent DNNs` by Awni Y. Hannun and al (https://arxiv.org/abs/1408.2873). The implementation keep tracks of the blank and non-blank probabilities. It also supports n-gram scoring on words and SentencePiece tokens. The input is expected to be a log-probabilities tensor of shape [batch, time, vocab_size]. Several heuristics are implemented to speed up the decoding process: - pruning of the beam : the beams are pruned if their score is lower than the best beam score minus the beam_prune_logp - pruning of the tokens : the tokens are pruned if their score is lower than the token_prune_min_logp - pruning of the history : the beams are pruned if they are the same over max_ngram history - skipping of the blank : the frame is skipped if the blank probability is higher than the blank_skip_threshold Note: The CTCPrefixBeamSearcher can be more unstable than the CTCBeamSearcher or the TorchAudioCTCPrefixBeamSearch searcher. Please, use it with caution and check the results carefully. Note: if the Acoustic Model is not trained, the Beam Search will take a lot of time. We do recommend to use Greedy Search during validation until the model is fully trained and ready to be evaluated on test sets. Note: This implementation does not provide the time alignment of the hypothesis. If you need it, please use the CTCBeamSearcher. Arguments --------- see CTCBaseSearcher, arguments are directly passed. Example ------- >>> import torch >>> from speechbrain.decoders import CTCPrefixBeamSearcher >>> probs = torch.tensor([[[0.2, 0.0, 0.8], ... [0.4, 0.0, 0.6]]]) >>> log_probs = torch.log(probs) >>> lens = torch.tensor([1.0]) >>> blank_index = 2 >>> vocab_list = ['a', 'b', '-'] >>> searcher = CTCPrefixBeamSearcher(blank_index=blank_index, vocab_list=vocab_list) >>> hyps = searcher(probs, lens) Frrr8rc Cs|jdur\g}|D]P}||j|j}|tdid|jd|ddd|jd|jd|j d |j d |j d |j d |j d |jd|jd|jd|jd|jd|jq |Sg}|D]}||j|j}||f}||vr||jdf\} } |jj| |j|d\} } | | } | | f||<||\}}|j}t|dkr||vr|j|||<|||7}|tdid|jd|ddd|jd|jd|j d |j d |j d |j d |j d |jd|jd|jd|jd|jd|j|q`|S)r9NrrrrrrrrrrrrrrrrrFr:rr*)rrrrrrrrrrrrrrrrrrrrr<r=r*r*r+rs                  z"CTCPrefixBeamSearcher.get_lm_beamsrN new_prefix new_tokennew_token_indexr previous_beamc Cs|D]}|j|kr|r||jkr||_|Sq|jsG||jkrG|jdkr(|jn|j|jg} t||j|jd||| dt j t j t j d } n|jr|dd|j kr|dd} |jdkra|jn|j|jg} |jd| }t||j|j| ||| ||dft j t j t j d } na||j kr|d} ||j kr|jn|jd| f} t||jd|j|||j| t j t j t j d } n0|jddkr||dfn|jd|df} t||jd|j||||j| t j t j t j d } || |r|j| _| S)a]Create a new beam and add it to the list of beams. Arguments --------- frame_index : int The index of the current frame. new_prefix : str The new prefix. new_token : str The new token. new_token_index : int The index of the new token. beams : list The list of beams. p : float The probability of the new token. previous_beam : CTCBeam The previous beam. Returns ------- new_beam : CTCBeam The new beam. rrr-Nr rr)rrrrrrrrrrrrrrr)r%rNr[r\r]rrr^rrXnew_beamrWrUrVr*r*r+ _get_new_beamjs"             z#CTCPrefixBeamSearcher._get_new_beamrrrrrc s|d|}t||dD]\}}|jjkrq |} tt|jkd| hB} |} | tt t j @} | D]p} || } j | }| D]b}|j |j }}| jkrdt|j|j| |_qJ||jkrst|j|| |_|j|}j|||| || |d}|j}| |jkr|tj krt||| }n| |jkrt||j| }||_qJq=|D]}|q|||}tdd|Dfdd|D}|}jrjdurdnjj}j||d }q d d|D}q |S) rDNrEr)rr^cSr r*rr r*r*r+rw^r z:CTCPrefixBeamSearcher.partial_decoding..crr*rr rr*r+rw_rr rFcSrGr*rHr r*r*r+rwjrI) rJrrrKrrrLrcopyr;rrrrrrrrrrr`rrrrrr<rrrrMr)r%rrrrrrrNrOrPrQ curr_beamsrRrSrTrrrrr_rrr+rr*rr+rsx #        "   z&CTCPrefixBeamSearcher.partial_decodingNrYro)rqrrrsrtrrr5rrr?rrr`r r4rr*r*r*r+rZs\4 l rZc%@seZdZdZdddddddddedddddd d d fd eeefd eedeedeede de dee dededededede deee fdeee fdede def$ddZ d'd e j d!ee j dfd"eeefd#d$Z d'd e j d!ee j dfd"eeefd%d&ZdS)(TorchAudioCTCPrefixBeamSearcheraJTorchAudio CTC Prefix Beam Search Decoder. This class is a wrapper around the CTC decoder from TorchAudio. It provides a simple interface where you can either use the CPU or CUDA CTC decoder. The CPU decoder is slower but uses less memory. The CUDA decoder is faster but uses more memory. The CUDA decoder is also only available in the nightly version of torchaudio. A lot of features are missing in the CUDA decoder, such as the ability to use a language model, constraint search, and more. If you want to use those features, you have to use the CPU decoder. For more information about the CPU decoder, please refer to the documentation of TorchAudio: https://pytorch.org/audio/main/generated/torchaudio.models.decoder.ctc_decoder.html For more information about the CUDA decoder, please refer to the documentation of TorchAudio: https://pytorch.org/audio/main/generated/torchaudio.models.decoder.cuda_ctc_decoder.html#torchaudio.models.decoder.cuda_ctc_decoder If you want to use the language model, or the lexicon search, please make sure that your tokenizer/acoustic model uses the same tokens as the language model/lexicon. Otherwise, the decoding will fail. The implementation is compatible with SentencePiece Tokens. Note: When using CUDA CTC decoder, the blank_index has to be 0. Furthermore, using CUDA CTC decoder requires the nightly version of torchaudio and a lot of VRAM memory (if you want to use a lot of beams). Overall, we do recommend to use the CTCBeamSearcher or CTCPrefixBeamSearcher in SpeechBrain if you wants to use n-gram + beam search decoding. If you wants to have constraint search, please use the CPU version of torchaudio, and if you want to speedup as much as possible the decoding, please use the CUDA version. Arguments --------- tokens : list or str The list of tokens or the path to the tokens file. If this is a path, then the file should contain one token per line. lexicon : str, default: None Lexicon file containing the possible words and corresponding spellings. Each line consists of a word and its space separated spelling. If None, uses lexicon-free decoding. (default: None) lm : str, optional A path containing KenLM language model or None if not using a language model. (default: None) lm_dict : str, optional File consisting of the dictionary used for the LM, with a word per line sorted by LM index. If decoding with a lexicon, entries in lm_dict must also occur in the lexicon file. If None, dictionary for LM is constructed using the lexicon file. (default: None) topk : int, optional Number of top CTCHypothesis to return. (default: 1) beam_size : int, optional Numbers of hypotheses to hold after each decode step. (default: 50) beam_size_token : int, optional Max number of tokens to consider at each decode step. If None, it is set to the total number of tokens. (default: None) beam_threshold : float, optional Threshold for pruning hypothesis. (default: 50) lm_weight : float, optional Weight of language model. (default: 2) word_score : float, optional Word insertion score. (default: 0) unk_score : float, optional Unknown word insertion score. (default: float("-inf")) sil_score : float, optional Silence insertion score. (default: 0) log_add : bool, optional Whether to use use logadd when merging hypotheses. (default: False) blank_index : int or str, optional Index of the blank token. If tokens is a file path, then this should be an str. Otherwise, this should be a int. (default: 0) sil_index : int or str, optional Index of the silence token. If tokens is a file path, then this should be an str. Otherwise, this should be a int. (default: 0) unk_word : str, optional Unknown word token. (default: "") using_cpu_decoder : bool, optional Whether to use the CPU searcher. If False, then the CUDA decoder is used. (default: True) blank_skip_threshold : float, optional Skip frames if log_prob(blank) > log(blank_skip_threshold), to speed up decoding (default: 1.0). Note: This is only used when using the CUDA decoder, and it might worsen the WER/CER results. Use it at your own risk. Example ------- >>> import torch >>> from speechbrain.decoders import TorchAudioCTCPrefixBeamSearcher >>> probs = torch.tensor([[[0.2, 0.0, 0.8], ... [0.4, 0.0, 0.6]]]) >>> log_probs = torch.log(probs) >>> lens = torch.tensor([1.0]) >>> blank_index = 2 >>> vocab_list = ['a', 'b', '-'] >>> searcher = TorchAudioCTCPrefixBeamSearcher(tokens=vocab_list, blank_index=blank_index, sil_index=blank_index) # doctest: +SKIP >>> hyps = searcher(probs, lens) # doctest: +SKIP Nr 2rrz-infFzTrtokenslexiconrlm_dictrrFbeam_size_tokenbeam_threshold lm_weight word_score unk_score sil_scorelog_addr sil_indexunk_wordusing_cpu_decoderrcCs||_||_||_||_||_||_||_||_| |_| |_ | |_ | |_ | |_ ||_ ||_||_||_||_|jrzddlm}Wn tyLtdwt|jtrZ|j }|j}n |j|j }|j|j}|did|jd|jd|jd|jd|jd |jd |jd |jd |jd |j d|j d|j d|j d|d|d|j|_dSzddlm}Wn tytdw|j dksJd||j|j|j|jd|_dS)Nr) ctc_decoderzTctc_decoder not found. Please install torchaudio and flashlight to use this decoder.rfrerrgnbestrFrhrirjrkrlrmrn blank_token sil_tokenrp)cuda_ctc_decoderz`cuda_ctc_decoder not found. Please install the latest version of torchaudio to use this decoder.z=Index of blank token has to be 0 when using CUDA CTC decoder.)rersrFrr*)rfrerrgrrFrhrirjrkrlrmrnrrorprqrtorchaudio.models.decoderrrrr}r _ctc_decoderrv)r%rerfrrgrrFrhrirjrkrlrmrnrrorprqrrrrtrurvr*r*r+r,s             z(TorchAudioCTCPrefixBeamSearcher.__init__rrrc sb|dur |d|}ntj|dg|d|jtjd}|jtjkr+|tj}|jtjkr5tdj r?|j r?| }j rI|j rI| }| sQt d||g}g}g}g}ttD]j rfddttD}fd d|D}||fd dttD}||n0d dttD}||fd dttD}fd d|D}||fddttD} || fddttD} || qeg} tt|||D]+\} \} }}| gt| ||D]\}}}| | td|d|||dqq| S)a-Decode log_probs using TorchAudio CTC decoder. If `using_cpu_decoder=True` then log_probs and wav_len are moved to CPU before decoding. When using CUDA CTC decoder, the timestep information is not available. Therefore, the timesteps in the returned hypotheses are set to None. Make sure that the input are in the log domain. The decoder will fail to decode logits or probabilities. The input should be the log probabilities of the CTC output. Arguments --------- log_probs : torch.Tensor The log probabilities of the input audio. Shape: (batch_size, seq_length, vocab_size) wav_len : torch.Tensor, default: None The speechbrain-style relative length. Shape: (batch_size,) If None, then the length of each audio is assumed to be seq_length. Returns ------- list of list of CTCHypothesis The decoded hypotheses. The outer list is over the batch dimension, and the inner list is over the topk dimension. Nr r)rr.zlog_probs must be float32.zlog_probs must be contiguous.cg|] }|jqSr*)rerrvjrRresultsr*r+rw`z@TorchAudioCTCPrefixBeamSearcher.decode_beams..cg|] }fdd|DqS)cg|]}j|qSr*rervrTrr*r+rwerIKTorchAudioCTCPrefixBeamSearcher.decode_beams...r*rvrerr*r+rwdcryr*) timestepsrrzr|r*r+rwir~cSsg|]}dqSrxr*)rvrVr*r*r+rwqscg|] }|jqSr*rrzr|r*r+rwtcr)crr*rrrr*r+rwvrIrr*rrr*r+rwurcrr*)wordsrzr|r*r+rwzrcrr*rrzr|r*r+rw}rr)rrrrr)rr tensorrint32r.tofloat32rrqis_cudar# is_contiguous RuntimeErrorrxr;rrrJrrr)r%rr tokens_preds words_preds scores_predstimesteps_predspredsrrrr&r$ batch_text batch_scorebatch_timestepsrrtimestepr*)rRr}r%r+r'#s                     z,TorchAudioCTCPrefixBeamSearcher.decode_beamscCs |||S)avDecode log_probs using TorchAudio CTC decoder. If `using_cpu_decoder=True` then log_probs and wav_len are moved to CPU before decoding. When using CUDA CTC decoder, the timestep information is not available. Therefore, the timesteps in the returned hypotheses are set to None. Arguments --------- log_probs : torch.Tensor The log probabilities of the input audio. Shape: (batch_size, seq_length, vocab_size) wav_len : torch.Tensor, default: None The speechbrain-style relative length. Shape: (batch_size,) If None, then the length of each audio is assumed to be seq_length. Returns ------- list of list of CTCHypothesis The decoded hypotheses. The outer list is over the batch dimension, and the inner list is over the topk dimension. r()r%rrr*r*r+r)s z(TorchAudioCTCPrefixBeamSearcher.__call__rx)rqrrrsrtrrr~rrr?r3r,r r4rrr'r)r*r*r*r+rcosY         ^   r  rc)r )$rtrrrr! itertoolsrtypingrrrrrrr$rr speechbrain.dataio.dataior speechbrain.utils.loggerr rqrr rr dataclassrrrnnModulerr7rZrcr*r*r*r+sH       %-qkN$