o }oi k@s>ddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z m Z mZddlZddlmZddlmZddlmmZddlmZdd lmZmZdd lmZmZdd l m!Z!m"Z"dd l#m$Z$dd l%m&Z&ddl'm(Z(m)Z)   dMde!de*de*de*de*de+de e,de e+de ej-fddZ.  dNde!de/de*de*dede*dee e*e*fde+de e,de ej-fddZ0   !dOde"dede*dee e*e*fde+de e,de/d"e1de ej-fd#d$Z2d%e e+de ej-fd&d'Z3d(ed)ej4de ee+ffd*d+Z5d(ede e e+e1ffd,d-Z6dPd.e+d/e1de e7fd0d1Z8d2e+d3e,de,fd4d5Z9d(ed6e+defd7d8Z:d"e7fd9d:Z;  ! !dQd3ee ej-e e ej-e e+fd(ed6e+de e+d;e1d"e1de e+e+ffdd?d@ggdAdfd2e+dBe+dCe+dDe e+dEe e+dFe+de7fdGdHZ=GdIdJdJZ>eGdKdLdLZ?dS)RN) dataclass)Path)NamedTemporaryFile)ListOptionalTupleUnion) DictConfig)tqdm)word_error_rate)ASRModelEncDecMultiTaskModel)manifest_utils rnnt_utils) FrameBatchASRFrameBatchMultiTaskAED)OccurancePunctuationErrorRate) get_full_path)logging model_utilscpuasrtokens_per_chunkdelaymodel_stride_in_secs batch_sizemanifest filepaths acceleratorreturnc Csg}g} |r |r td|dur|durtd|r`g}t|ddd5} td| D]&} | } | s3q*t| } t| d|d } || d | vrP| | d q*Wdn1s[wYt tj |d krnd nd g}d |_ t tt|dt|dD]7}|||t||krdd|D}||||||||}||||j |7_ qt|d krt||_t||j_|dd|D}|||||||}||||j t|7_ Wdn 1swYWdn 1swYtjdddvrMt| d kr8td|D]}td|q.nt|| D]\}}td|td|q=t|}|S)z Moved from examples/asr/asr_chunked_inference/rnnt/speech_to_text_buffered_infer_rnnt.py Write all information presented in input manifest to output manifest and removed WER calculation. *Please select either filepaths or manifestN.Either filepaths or manifest shoud not be Nonerutf_8encodingzParsing manifest files...audio_filepath audio_file manifest_filetextrcudarSample:)desctotalcSg|]}|qSr0.0sampler0r0e/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/asr/parts/utils/transcribe_utils.py Qz/get_buffered_pred_feat_rnnt..cSr/r0r0r1r0r0r4r5`r6DEBUG01yt#ground-truth text does not present!hyp:ref:) ValueErroropenprintstripjsonloadsrappendtorchinference_modeampautocast sample_offsetr rangelenresetread_audio_file transcribeextendclearrframe_buffererosenvirongetzipwrap_transcription)rrrrrrrrhypsrefsmfst_fLrowr(batchidx audio_fileshyp_listhypref wrapped_hypsr0r0r4get_buffered_pred_feat_rnnt%s|            re frame_lenpreprocessor_cfgdevicec Csd|_tjj|} | |g} g} |r|rtd|dur'|dur'td|rIt|ddD]} || | ||| ||} | | q/nUt |ddd F}t|ddD]7} || } | sdqWt| }d |vrt| |d t|d |d }| |||| ||} | | qWWdn1swYtjd ddvrt| dkrtd| D]} td| qnt| | D]\} }td| td|qt| }|S)z Moved from examples/asr/asr_chunked_inference/ctc/speech_to_text_buffered_infer_ctc.py Write all information presented in input manifest to output manifest and removed WER calculation. Noner Nr!r,)r-r"r#r$r*r&r'r7r8r9rr=r>r?) normalizenemo_asrmodelsEncDecCTCModelBPEfrom_config_dicttor@r rNrOrPrFrArCrDrErrTrUrVrMrBrWrX)rrfrrrgrrhrr preprocessorrYrZr\rbr[r]r(rcrdr0r0r4get_buffered_pred_featvsT          rqF timestampsc Csd|_t|}||g} g} |r|rtd|dur%|dur%td|r_tdt|dt|ddD]'} | dd d d d d |rCd nd d} | |j | ||| d| } | | q6net |dddV}t|}t|dt|ddD]=}| |}|sqwt|}|rd |d<d|vr| |dt|d|d} |j | |||d| } | | qwWdn1swYt| }|S)Nrir r!zDeteced audio files as input, default to English ASR with Punctuation and Capitalization output. Please use manifest input for other options.z Transcribing:P)r-r.ncolsienryesnothingno)r&duration source_langtaskname target_langpncanswer timestamp) meta_datar"r#r$rr*r&r')rjr rnror@rinfor rMrNrOrPrFrAlist readlinesrCrDrErrX)rrgrrhrrrrsrprYrZr(metarbfinlinesliner3rdr0r0r4#get_buffered_pred_feat_multitaskAEDsb        rrYcCs@t|dtjr |Sg}|D]}tjdg|d}||q|S)zEWrap transcription to the expected format in func write_transcriptionrrr)score y_sequencer*) isinstancer HypothesisrF)rYrdrb hypothesisr0r0r4rXs rXcfg map_locationcCs|jdur9|jdkr9tj|jdd}|j}t|}td|j|j|j|d}t j t j |jd}n tj |j|d}|j}t|d rat|d ra|j|jjd d|jjd dd ||fS)zBSetup model from cfg and return model and model name for next stepNriT) restore_path return_configzRestoring model : )rrr) model_namer model_changechange_attention_modelself_attention_modelatt_context_size)rr) model_pathr restore_fromtargetrimport_class_by_pathrr__name__rTpathsplitextbasenamefrom_pretrainedpretrained_namehasattrrr conformerrV)rr model_cfg classpathimported_class asr_modelrr0r0r4 setup_models* rc Csd}d}|jdur#|js#ttjtj|jd|jdd}||fSg}t|j j dkr:t d|j ddS| dd }t|j d 9}|D].}|}|sRqIt|}t|||j ||<| d durw|jrwtd |d |j dqIWdn1swYtdddd/}t|j |jdD]}t|||j d}||d <|||t|dqWdn1swY|j}||fS)a] Prepare audio data for transcription. Args: cfg (DictConfig): Configuration dictionary containing the following parameters: - audio_dir (str): Path to the directory containing audio files. - append_pred (bool): Flag indicating whether to append predictions to an existing dataset. - audio_type (str): Type of audio files to consider. - dataset_manifest (str): Path to the dataset manifest file. - audio_key (str, optional): Key in the manifest file specifying the audio file path. Defaults to 'audio_filepath'. - presort_manifest (bool, optional): Flag indicating whether to presort the manifest file. Defaults to True. Returns: Tuple[List[str], bool]: A tuple containing the following: - filepaths (List[str]): List of filepaths to the audio files if path to the directory containing audio files is provided. - sorted_manifest_path (bool): Path to the sorted manifest file if path to the dataset manifest file is provided. Nz**/*.T) recursiverzThe input dataset_manifest z is empty. Exiting! audio_keyr&rtrzz*Requested presort_manifest=True, but line z in manifest z6 lacks a 'duration' field.w.jsonF)modesuffixdelete)try_sortr' ) audio_dir append_predrglobrTrjoin audio_typestatdataset_manifestst_sizererrorrVrArCrDrErpresort_manifestr@rread_and_maybe_sort_manifestrFwritedumpsname) rrsorted_manifest_pathrfhritemfr(r0r0r4prepare_audio_data&sD&    rrrcCs6t|}|rtdd|Drt|dddd}|S)zDSorts the manifest if duration key is available for every utterance.css$|] }d|vo |dduVqdS)rzNr0r2rr0r0r4 cs"z/read_and_maybe_sort_manifest..TcSs|dS)Nrzr0)rr0r0r4dsz.read_and_maybe_sort_manifest..reversekey)r read_manifestallsorted)rritemsr0r0r4r`s r manifest_pathtranscriptionsc Cst|dd}ddt|D}Wdn1swYtdd|Ds*|Sddt|dd d d D}~t|d t}|rHtt|}dgt|}t|D] \}}||||<qS|ritt tt|}|S) Nutf-8r$cSs*g|]\}}|dkr|t|fqSrCrDrE)r2r_lr0r0r4r5js*z/restore_transcription_order..css,|]}d|dvo|ddduVqdS)rzNr0rr0r0r4rks*z.restore_transcription_order..cSsg|]}|dqS)rr0rr0r0r4r5mTcSs |ddS)Nrrzr0)itr0r0r4rms z-restore_transcription_order..rr) rA enumeraterrrrrWrMtuplemap) rrrrnew2oldis_list reorderednewoldr0r0r4restore_transcription_orderhs  rrcCsz|jdur;|jdurtjtj|jdd|_|S|jdur/|jdd|jd|_|S|jdd|d|_|S)z2Compute filename of output manifest and update cfgN.r_) output_filenamerrTrdirnamerpred_name_postfixrreplace)rrr0r0r4compute_output_filenamezs   rcCsFtt|D]}t||d||d<t||d||d<q|S)a Normalize the dictionary of timestamp values to JSON serializable values. Expects the following keys to exist - "start_offset": int-like object that represents the starting index of the token in the full audio after downsampling. "end_offset": int-like object that represents the ending index of the token in the full audio after downsampling. Args: timestamps: Nested dict. Returns: Normalized `timestamps` dictionary (in-place normalized) start_offset end_offset)rLrMint)rsval_idxr0r0r4normalize_timestamp_outputsr compute_langsc Csx|jrtd|jd|jdur|j}n|}d|}nd}d}t|dtr-|} d}nct|dtjrA|} |j j j s@Jd nOt|dt rt|ddtjrgg} } |D]3} | | d|j j j sg} | D]} t| jtjr{| jn| j}| | j|fqk| | qYntt|jjjddd t|jd d d d }|jdurt| D]`\}}|sd||||i}nGd||||ji}|r|j}|durt|tr|dd|D] }t||}|||<q|r|j |d<|j!|d<|j j j s| ||d<|"t#$|d qnt|j%dd d}t|D]w\}}|&}|s,qt#'|}|s;| |||<nP| |j||<|rn| |j}|durnt|trn|dd|D]}t||}|||<q_|r| |j |d<| |j!|d<|j j j s| ||d<|"t#$|d qWdn 1swYWdn 1swY|j|fS)z-Write generated transcription to output file.z Transcripts will be written in "z" fileN pred_text_ pred_textTrFz+Works only with return_best_hypothesis=true)parentsexist_okrrr)r%newliner&timestep pred_langpred_lang_charsbeamsr"r$)(rrrrrrstrrrdecodingbeamreturn_best_hypothesisrrFrrGTensornumpyrr* TypeErrorrparentmkdirrArrrdictpopkeysrlangs langs_charsrrDrrrCrE)rrrrrrspred_by_model_namepred_text_attr_namereturn_hypotheses best_hypsrrYrrbrrr_ transcriptionrrvaluesfrrr0r0r4write_transcriptions     "                 6rr*rwer)r,?reference_fieldhypothesis_fieldmetricspunctuation_marksoutput_manifest_pathcCsHgd}t|dkrtd|d|D]}||vr%td|d|dqd|vr9t|dkr4tdt|d }d |v} d |v} d|v} t|d } | } d d| D}g}tdd|dt|D]u}||}||}| rt |g|gdd}t d|d|d <| rt |g|gdd}t d|d|d <| r|j ||d\}}}t d|j d|d<t d|j d|d<t d|jd|d<t d|jd|d<t d|jd|d<||qhWdn1swY|dur"t|d}|D]}t|}||dqWdn 1swYtd ||S)!a Computes metrics per sample for given manifest Args: manifest_path: str, Required - path to dataset JSON manifest file (in NeMo format) reference_field: str, Optional - name of field in .json manifest with the reference text ("text" by default). hypothesis_field: str, Optional - name of field in .json manifest with the hypothesis text ("pred_text" by default). metrics: list[str], Optional - list of metrics to be computed (currently supported "wer", "cer", "punct_er") punctuation_marks: list[str], Optional - list of punctuation marks for computing punctuation error rate ([".", ",", "?"] by default). output_manifest_path: str, Optional - path where .json manifest with calculated metrics will be saved. Returns: samples: dict - Dict of samples with calculated metrics )rcerpunct_errzL'metrics' list is empty. Select the metrics from the supported: r'zK' metric is not supported. Currently supported metrics are rzHpunctuation_marks list can't be empty when 'punct_er' metric is enabled.)rrrr"cSs"g|] }|dkrt|qSrrr2rr0r0r4r56s"z.compute_metrics_per_sample..z Computing z, z per sampleF) hypotheses referencesuse_cerdT) referencerpunct_correct_ratepunct_deletions_ratepunct_insertions_ratepunct_substitutions_ratepunct_error_rateNrrzOutput manifest saved: )rMAssertionErrorrrArrrrr r roundcompute correct_ratedeletions_rateinsertions_ratesubstitutions_raterrFrDr writelines)rrrrrrsupported_metricsmetricoper_objuse_werr use_punct_errrsamplessamples_with_metricsr3r r sample_wer sample_ceroperation_amountssubstitution_amountspunctuation_ratesoutputrr0r0r4compute_metrics_per_samplesr          r;c@sheZdZdefddZdeedeefddZdeedeefdd Zdeedeefd d Zd S) PunctuationCapitalizationrcCs8|rtdd|d|_td|_dSd|_dS)a Class for text processing with punctuation and capitalization. Can be used with class TextProcessingConfig. Args: punctuation_marks (str): String with punctuation marks to process. Example: punctuation_marks = '.,?' z([rz])z\s{2,}N)recompilerregex_punctuationregex_extra_space)selfrr0r0r4__init__^s z"PunctuationCapitalization.__init__rrc jdurfdd|DS|S)Nc s(g|]}jdjd|qS) z \1 r@subr?rCrrAr0r4r5nszBPunctuationCapitalization.separate_punctuation..r?rArr0rGr4separate_punctuationls  z.PunctuationCapitalization.separate_punctuationcCsdd|DS)NcSsg|]}|qSr0)lowerrr0r0r4r5urz:PunctuationCapitalization.do_lowercase..r0rIr0r0r4 do_lowercasetsz&PunctuationCapitalization.do_lowercasecrC)Nc s(g|]}jdjd|qS)rDrErrGr0r4r5ys(z.rHrIr0rGr4rm_punctuationws z(PunctuationCapitalization.rm_punctuationN) r __module__ __qualname__rrBrrJrLrMr0r0r0r4r<]s r<c@s>eZdZUdZeed<dZeed<dZeed<dZ eed<dS) TextProcessingConfigrrFrLrMTrJN) rrNrOrr__annotations__rLboolrMrJr0r0r0r4rP~s   rP)NNr)NN)NNrrF)F)NFF)@rrDrTr= dataclassesrpathlibrtempfilertypingrrrrrG omegaconfr tqdm.autor nemo.collections.asr collectionsrrk nemo.collections.asr.metrics.werr nemo.collections.asr.modelsr r nemo.collections.asr.parts.utilsrr0nemo.collections.asr.parts.utils.streaming_utilsrr(nemo.collections.common.metrics.punct_err4nemo.collections.common.parts.preprocessing.manifestr nemo.utilsrrrrrrrefloatrqrRrrXrhrrrrrrrrr;r<rPr0r0r0r4s           Y  F E :    f ]!