o irN@s,ddlZddlZddlZddlZddlmZddlmZmZm Z m Z m Z ddl m Z ddlmZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlm Z ddl!m"Z"ddl#m$Z$ddl%m&Z&m'Z'ddl(m)Z)ddl*Z*e +ddGdddej,j-Z.dS)N)autocast)UnionDictListTupleOptional)tables)CTC)postprocess_utils) th_accuracy) to_device) DatadirWriter) Hypothesis)mae_loss)force_gatherable)LabelSmoothingLoss) add_sos_eos) make_pad_mask)ts_prediction_lfr6_standard)load_audio_text_image_video extract_fbank) pad_sequence model_classesParaformer_v2_communityc+seZdZdZ                  dBd eed eed edeededeededeededeededededededededede de de f*fd d! Z d"e j d#e j d$e j d%e j d&e e j eee j fe j ff d'd(Zd"e j d#e j d&e e j e j ffd)d*Zd+e j d,e j d-e j d.e j fd/d0Zd+e j d,e j d-e j d.e j fd1d2Zd3d4ZdCd5e j d6e j d&efd7d8Zd9d:Zd;d<Z    dDd=efd>d?Zd@dAZZS)E Paraformerz Author: Speech Lab of DAMO Academy, Alibaba Group Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition https://arxiv.org/abs/2206.08317 N?PrFspecaug specaug_conf normalizenormalize_confencoder encoder_confdecoder decoder_confctcctc_conf ctc_weight input_size vocab_size ignore_idblank_idsoseos lsm_weightlength_normalized_lossshare_embeddinguse_1st_decoder_lossc st|durtj|}|di|}|dur'tj|}|di|}tj|}|dd| i|}|}|durNtj|}|d| |d|}| dkrb| durXi} t d| |d| } ||_ |durk|n| d|_ |durv|n| d|_ | |_ ||_| |_||_||_||_| dkrd|_n||_t| |||d|_| dkrd|_n| |_||_|jrd|j_||_||_d|_d|_dS) Nr,)r-encoder_output_sizer )odimr6rg?)size padding_idx smoothingnormalize_length)super__init__rspecaug_classesgetnormalize_classesencoder_classes output_sizedecoder_classesr r/r0r1r-r.r+r!r#r%r'r criterion_attr)r4embedr5r3 beam_searcherror_calculator)selfr!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5kwargs specaug_classnormalize_class encoder_classr6 decoder_class __class__r<_/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/paraformer_v2_community/model.pyr>&sd "       zParaformer.__init__speechspeech_lengthstext text_lengthsreturncKsdt|dkr|dddf}t|dkr |dddf}|jd}|||\}}d\} } d} t} |jdkrU|||||\} } | durM| nd| d<| | d<|||||\} }}}|jdkri| }n |j| d|j| }| dur}| nd| d<|| d <|| d <|| d <t || d <|| d <|j r| }t || |f|j\}} }|| |fS)zEncoder + Decoder + Calc loss Args: speech: (Batch, Length, ...) speech_lengths: (Batch, ) text: (Batch, Length) text_lengths: (Batch,) rNrNNr loss_ctccer_ctcloss_attacccerwerloss batch_size)lenr8shapeencodedictr+_calc_ctc_lossdetach_calc_att_losstorchcloner3sumrdevice)rIrRrSrTrUrJr_ encoder_outencoder_out_lensrXrYloss_prestatsrZacc_attcer_attwer_attr^weightr<r<rQforwardsD     zParaformer.forwardcKstd%|jdur|jr|||\}}|jdur"|||\}}Wdn1s,wY|||\}}}t|trC|d}||fS)zEncoder. Note that this method is used by asr_inference.py Args: speech: (Batch, Length, ...) speech_lengths: (Batch, ) ind: int FNr)rr!trainingr#r% isinstancetuple)rIrRrSrJrkrl_r<r<rQrbs   zParaformer.encoderkrlys_pad ys_pad_lensc Csd}|d}|j|}g}t|D]V} || d|| f} || d|| f} t&| } |j| | |j d} | |j } | | |j }Wdn1sVwY| | ||| }||qt|dd |j }|||||}|d|d}}|dur|}|||}t|d|j||jd}|js|jdurd\}}n|jdd }|||\}}||||fS) Nr)r/T) batch_firstrr) ignore_labelrWdim)r8r)softmaxrangergno_gradlog force_aligncpur/torjmap_alignment_to_target_indexaverage_repeats_trainingappendrr'rEr viewr-r.rtrHargmax)rIrkrlrxrydecoder_out_1str_ ctc_probs_allcompressed_ctc_listb ctc_prob_btext_bctc_log_prob_b align_pathtarget_idx_pathctc_comppadded_ctc_input decoder_outs decoder_outrwrZrorprqys_hatr<r<rQrfs>           zParaformer._calc_att_losscCsR|||||}d}|js%|jdur%|j|j}|j||dd}||fS)NT)is_ctc)r)rtrHrdatar)rIrkrlrxryrXrYrr<r<rQrd3s zParaformer._calc_ctc_losscCsP||k}t|d}||d<|||k@}tj|ddd}t||d}|S)a Robustly map CTC alignment path (Token IDs) to Target Indices. Logic: Detect boundaries where a new token segment begins. A segment starts if the current frame is a Token AND it is different from the previous frame (considering CTC topology where repeats are separated by blanks or are distinct tokens). Example: Text: [A, B] Align Path: [A, A, _, B, B] Output: [0, 0, -1, 1, 1] rrr|r)rgrollcumsumlongwhere)rIrr/is_token prev_pathnew_segment_start segment_idsrr<r<rQrDs  z(Paraformer.map_alignment_to_target_index ctc_probsycCs6|d}|d}tjj|||d\}}|dS)actc forced alignment. Args: torch.Tensor ctc_probs: hidden state sequence, 2d tensor (T, D) torch.Tensor y: id sequence tensor 1d tensor (L) int blank_id: blank symbol index Returns: torch.Tensor: alignment result N)blankr)r torchaudio functional forced_align)rIrrr/ alignmentsrwr<r<rQres zParaformer.force_alignc Cs|}|d}tj||f|j|jd}tj|df|j|jd}|dk}||} ||} | dkr3|S| dd|} |d| | tj | ddf|jd} |d| d| ||d}|S)a Aggregates frames belonging to the same target index using scatter_add. Args: ctc_probs: [T, V] target_idx_path: [T], values in [-1, 0, ... U-1] target_len: U Returns: compressed: [U, V] r)rjdtyperrrjg& .>) r8rgzerosrjrnumel unsqueezerepeat scatter_add_ones) rIrr target_lenUV compressedcountsmask valid_indices valid_probsindex_expandedrr<r<rQrts   z#Paraformer.average_repeats_trainingc Cs|dkrtjd|df|jdStj|dd\}}tj|dd}ttjdg|jd|ddg}g}t |D]$\}} | |j kr_|| } || } || | j dd} | | q;|sotjd|df|jdSt|S) z Returns: merged_probs: [U', V] timestamps: List[Tuple[int, int]] -> [(start_frame, end_frame), ...] rrrT) return_countsr|Nr)rrgrr8rjunique_consecutivercattensor enumerater/itemmeanrstack) rIr greedy_path unique_tokensr end_indices start_indices merged_probsitokenstartendavg_probr<r<rQaverage_repeats_inferences $     z$Paraformer.average_repeats_inferencekeyc! si}t|tjr7|dddkr7||}} t|jdkr'|dddddf}| dur1| d} nQ|jd} nKt} t ||j |dd|dd|d } t} | | d |d <t | |dd|d \}} t} | | d |d <| |j|jd|d<|j|dd}| j|dd} |ddr|}|| \}}t|tr|d}j|}|jdd}g}|\}}}t|dttfr|d}t||kr||}t|D]q}||d||f}||d||f}||}|ddkrg}q|d}tj|dg|jd}|||d|||d||\}}|jddd}|}tt fdd|}|||d} |!| q||fS)N data_typesoundfbankrrfsi>)raudio_fsr tokenizerz0.3f load_data)rfrontend extract_featibatch_data_timerjrfp16Frr|cs|jko|jko|jkS)N)r1r0r/)xrIr<rQsz&Paraformer.inference..)r token_int)"rurgTensorr@r`rasqueezetime perf_counterrrrrir frame_shiftlfr_nrhalfrbrvr)r~rr8listrrrrrjr'tolistfilterr)!rIdata_in data_lengthsrrrrJ meta_datarRrStime1audio_sample_listtime2time3rkrlrctc_greedy_pathsresultsr_ndrprobspathcompressed_probrcompressed_prob_inin_lensrrwyseqresult_ir<rrQ inferences                 zParaformer.inferencecKs2ddlm}d|vrd|d<|dd|i|}|S)Nr)export_rebuild_model max_seq_lenimodelr<) export_metar)rIrJrmodelsr<r<rQexports zParaformer.export)NNNNNNNNNNrrrrrrrr FFF)r)NNNN)__name__ __module__ __qualname____doc__rstrrfloatintboolr>rgrrrsrbrfrdrrrrrrr __classcell__r<r<rOrQrs     v D  6 !%$ br)/rcopyrgloggingtorch.cuda.amprtypingrrrrrfunasr.registerrfunasr.models.ctc.ctcr funasr.utilsr funasr.metrics.compute_accr funasr.train_utils.device_funcsr funasr.utils.datadir_writerr funasr.models.paraformer.searchr&funasr.models.paraformer.cif_predictorrr"funasr.losses.label_smoothing_lossr+funasr.models.transformer.utils.add_sos_eosr*funasr.models.transformer.utils.nets_utilsrfunasr.utils.timestamp_toolsrfunasr.utils.load_utilsrrtorch.nn.utils.rnnrrregisternnModulerr<r<r<rQs0