o |is @sdZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z m Z ddl mZddlZddlZddlmZmZddlmZdejd <dd d d d dddddddd Zddddddddddd d!d" ZeeZGd#d$d$ejjZ e!d%Z"e!d&Z#e!d'Z$e!d(Z%e!d)Z&e!d*Z'gd+Z(d,e)d-e)fd.d/Z*d,e)d-e)fd0d1Z+d,e)d-e)fd2d3Z,d,e)d-e)fd4d5Z-d6e)d7e)fd8d9Z.d:e)d;e)fde0d-e1fd?d@Z2d>e0d-e0fdAdBZ3d>e0d-e1fdCdDZ4dEe)dFe)fdGdHZ5dOdJdKZ6dLdMZ7e8dNkr e7dSdS)Pa Benchmark Maya ASR TDT 1.1B (Hybrid FastConformer RNNT/TDT) on indic-asr-benchmark-6k. Outputs: metrics.json, sample_analysis.json, error_analysis.json following BENCHMARK_SCHEMA.md v1 normalization. Usage: python3 benchmark_maya_asr_tdt.py --checkpoint /home/ubuntu/training/checkpoints/maya-asr-tdt-1.1b-ckpt-60000/model.ckpt --config /home/ubuntu/training/maya-asr-hybrid-fastconformer-rnnt-stage1/configs/train/stage1_prod_8xh200.yaml --checkpoint-name ckpt-60000 N)Counter defaultdict)datetimetimezone)Path)wercer) OmegaConf soundfileHF_AUDIO_BACKEND ) hibntatemrguknmlpaorasenrrrrrrrrrr r!r") hindibengalitamiltelugumarathigujaratikannada malayalampunjabiodiaassameseenglishcs<eZdZdedeffdd ZdejdejfddZZS) LanguageEmbedding num_languages embed_dimcs6ttj|||_tjjj|jjddddS)Ng{Gz?)meanstd) super__init__torchnn Embeddingembedinitnormal_weight)selfr0r1 __class__//home/ubuntu/training/benchmark_maya_asr_tdt.pyr61s zLanguageEmbedding.__init__lang_idsreturncCs||dS)Nr )r: unsqueeze)r>rCrArArBforward6zLanguageEmbedding.forward) __name__ __module__ __qualname__intr6r7TensorrF __classcell__rArAr?rBr/0sr/z([\u200b-\u200f\u2028-\u202f\ufeff\u00ad]z'[!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]z[\u0964\u0965\u0970\u0971]z)[\u2010-\u2027\u2030-\u205e\u2e00-\u2e4f]z>[\u2018\u2019\u201a\u201b\u201c\u201d\u201e\u201f\u00ab\u00bb]z \([^)]*\)) if i if i if i if i if textrDcCstd|S)NNFC) unicodedata normalizestriprNrArArBnorm_rawHrGrTcCstd|}td|}td|}d|}td|}|dddddd}t d|}t d|}t d|}| }d| }|S) NNFKC 'u—-u–u‒)rPrQZW_CHARSsub ANNOTATIONSjoinsplitQUOTESreplace PUNCT_COMMON PUNCT_INDICPUNCT_EXTENDEDlowerrRrSrArArB norm_standardLs       recCsJt|}tdd|}tD]}tdD]}|t||t|}qq |S)Nz (\d),(\d)z\1\2r)rerer[INDIC_DIGIT_OFFSETSranger`chrstr)rNoffsetirArArB norm_numcanon[s rmcCst|ddS)zBMER: norm_standard + remove ALL spaces -> single character stream.rWrV)rer`rSrArArBnorm_merdsrn ref_chars hyp_charsc stt|t|}fddt|dD}t|dD]}|||d<qtdD]}||d|<q+td|dD]G}tddD]=}||d||dkra||d|d|||<qDdt||d|d||d||||d|||<qDq;t}|}}|dks|dkr8|dkr|dkr||d||dkr|d8}|d8}n|dkr|dkr|||||d|ddkr||d|d8}|d8}nU|dkr|||||d|dkr||d|d8}n5|dkr-||||||ddkr-|dkr||dn |dkr(|d|d8}n |S|dks|dks|S)zCCompute Levenshtein DP and backtrace to find edit positions in ref.csg|] }dgdqS)rr rA).0_mrArB oz*_levenshtein_backtrace..r r)lenrhminsetadd)rorpndprljedited_ref_positionsrArsrB_levenshtein_backtraceksBB (4( ,    rref_normhyp_normc Cs|}|sdS|dd}|dd}t|}||kr d|fS|s$dS|s*||fSg}t|D]\}}|D]} ||q6q0t||} t} | D]} | t|krX| || qIt| |fS)zXCompute space_norm_wer for a single sample. Returns (error_words, total_words). rrrWrVr)r^r`rw enumerateappendrryrz) rr ref_words ref_nospace hyp_nospace total_words char_to_wordwiwordrredited_positions touched_wordsposrArArBcompute_space_norm_wer_samples0      rsamplesc/stt}|D] }||d|qigg}}gg}}gg}}gg} } d\} } t|D]Z} || }gg}}gg}}gg}}gg}}d\}}d}|D]}|d}|ddpbd}|sk|d7}t|t|}}t|t|}}t |t |} }!t |t |}"}#|sqU||||r|nd||r|nd||r|nd|| r| nd||!r|!nd||"r|"nd||#r|#ndt |r|nd|r|nd\}$}%||$7}||%7}qUt ||d }&t ||d }'t ||d }(t ||d })t ||d }*|dkr ||d nd }+t|t|&d t|'d t|(d t|+d t|)d t|*d |t|&|'d t|'|(d t|'|+d t|'|)d d d | <||||||||||||| || || |7} | |7} q2| dkr| | d nd },t|tt ||d d tt ||d d tt ||d d t|,d tt | | d d tt ||d d dd<ddD}-t|-}.|.ttfdd|-D|.d ttfdd|-D|.d ttfdd|-D|.d ttfdd|-D|.d ttfdd|-D|.d ttfdd|-D|.d dd<S)Nlanguagerr reference hypothesisrVr dr2r ) raw_to_normnorm_to_numcanonnorm_to_space_norm norm_to_mer) n_sampleswer_rawwer_norm wer_numcanonspace_norm_wermercer_normempty_hypothesesnormalization_delta)rrrrrrr __overall__cSsg|] }|ds|qS)rr startswithrqkrArArBrurvz/compute_metrics_for_samples..c3|] }|dVqdS)rNrArqlresultrArB z.compute_metrics_for_samples..c3r)rNrArrrArBrrc3r)rNrArrrArBrrc3r)rNrArrrArBrrc3r)rNrArrrArBrrc3r)rNrArrrArBrr) n_languagesrrrrrr __macro_avg__)rlistrsortedkeysgetrRrTrermrnr compute_wer compute_cerrwroundextendsum)/rby_langs all_ref_raw all_hyp_raw all_ref_norm all_hyp_norm all_ref_nc all_hyp_nc all_ref_mer all_hyp_merall_snw_errors all_snw_totallang lang_samplesrefs_rawhyps_raw refs_norm hyps_normrefs_nchyps_ncrefs_merhyps_merlang_snw_errorslang_snw_total empty_countrefhyprrhrrnhnrnchncrmhmewtww_raww_normw_ncw_merc_normw_snw overall_snw lang_keysn_langsrArrBcompute_metrics_for_sampless         $$$          rcCsg}|D]}|d}|ddpd}t|}t|}t|}t|}t|} t|} t|} t|} zNt| g| r=| ndgd} t|rH|ndg|rN|ndgd}t| rY| ndg| r_| ndgd}t|rj|nd|ro|nd\}}|dkr~||dnd}Wnt yd\} }}}Ynwg}||kr| d ||kr| d ||kr||kr| d |s| d t d |st d |r||kr| d|dkr| d||dkr| d| |d|d||||||| | t | dt |dt |dt |d|dq|S)NrrrVrrrr2)Y@rrr exact_matchexact_match_normpunctuation_only_diffempty_hypothesis\dnumeric_mismatchPhigh_werg? spacing_erroridrr )rrrrrr ref_numcanon hyp_numcanonref_merhyp_merrrrrflags)rrermrnrTrrRrr Exceptionrrfsearchr)routrrrrrrrrrrrrrrrrrrrArArBbuild_sample_analysis"sn ""          rc* s&tt}|D] }||d|qii}t|D]e}||}t}t}t}d} d} d} d} d} d}g}|D]}|d}|ddpId}t|}t|}|sZ|d7}zt |r`|ndg|rf|ndgd}Wn t yxd }Ynw||d |f| }| }t|}t|}|D]}||||d}|dkr|||7<q|D]}||||d}|dkr|||7<qt t |t |}t|D]}||||kr|||||fd7<qtd |std |r||kr| d7} ||kr||kr| d7} q<|jd d ddd|ddDddd}dd|ddD}dd|Ddd}|rQtdd|Dt |nd ||<dd|dDdd|dDdd|dD| | | | | |d|||dd|<qt|dd d} d d| ddDddd}!d!d| ddD}"tfd"dD}#tfd#dD}$t |}%|#|%d$krd%}&d&}'n|$|%d'krd(}&d)}'nd*}&d+}'|$|%d,krd-n |$|%d.krd/nd0}(|#|%d'krd-n |#|%d.krd/nd0})|&|'|(|)|!|"d1d2<S)3NrrrrrVr rrrrrcS|dSNr rAxrArArBz&build_error_analysis..)keycSg|]\}}|qSrArArqsidrrrArArBruz(build_error_analysis..cSrrArArrArArBrurrc Ss\g|]*}td|dstd|ddpdr,t|dt|ddp%dkr|dqS)rrrrVr)rfrrre)rqrrArArBrus css|]\}}|VqdSNrA)rqrrwrArArBrsz'build_error_analysis..cSs g|] \\}}}|||dqS))rrcountrA)rqrhcrArArBrus  cSg|] \}}||dqS)rr rArqr rrArArBrucSrrrArrArArBrur)numeric_mismatch_countpunctuation_only_countspacing_tokenization_countentity_mismatch_countscript_confusion_countempty_hypothesis_count) worst_samples best_samplesnumeric_mismatch_samples)top_substitutionstop_insertions top_deletions error_bucketsexamplescSrrrArrArArBrrcSrrArArqrrrrArArBrurcSrrArAr#rArArBrurc3*|]}|ds|ddVqdS)rrr!rNrrrrArBr(c3r$)rrr!rNrrrrArBrr%g333333?zformatting-limited formattingg?znumeric-limitednumericzrecognition-limited recognitiong333333?highg?moderatelow)model_diagnosisprimary_error_sourcenumeric_verbalization_impactformatting_impactworst_languagesbest_languages __summary__)rrrrrrrrerRrrr^rxrwrhrfrsortr most_commonitems)*rrrlang_wer_normsrrsubsinsdels numeric_mm punct_only spacing_tok entity_mm script_conf empty_hypper_sample_werrrrrswr hyp_words ref_counter hyp_counterr diffmin_lenrlworstbestnumeric_samples sorted_langs worst_langs best_langstotal_punct_only total_numeric total_samples diagnosisprimary_sourcenumeric_impactr/rArrBbuild_error_analysis_s&     (   ((  rS config_pathcheckpoint_pathcsBddlmm}td|t|}tt|j j j }|j j j }t j|s5t j|||j j _ t j|j j j sHJd|j j j td|j j j td}|s_|dt||j j_t||j j_d|j_d|j_td |jj|j dd td |ddl}ddl}ddlGd d d jddl m!} | j"d!fdd } | | _"z t#j|ddd} W| _"n| _"wt$t%d} | _&'d| j(| ddd\} }tdt)| dt)|d| rtd| ddj*j+fdd}|j*_+td,-.t#j/td S)"NrzLoading config from zTokenizer dir not found: zTokenizer dir: z/tmp/dummy_manifest.jsonlzB{"audio_filepath": "/dev/null", "text": "dummy", "duration": 1.0} r gpuz!Initializing model from config...)cfgtrainerzLoading checkpoint from cs$eZdZdZiZfddZZS)z"load_model.._SafeUnpicklerz5Unpickler that creates stubs for any missing classes.c snzt||WSttfy6|d|}||jvr/Gddd}||_||_||j|<|j|YSw)N.c@seZdZddZddZdS)z;load_model.._SafeUnpickler.find_class..Stubc_sdSr rA)r>akwrArArBr6szDload_model.._SafeUnpickler.find_class..Stub.__init__cSs$|jt|tr |dSidSr )__dict__update isinstancedict)r>staterArArB __setstate__s$zHload_model.._SafeUnpickler.find_class..Stub.__setstate__N)rHrIrJr6rarArArArBStubs rb)r5 find_classModuleNotFoundErrorAttributeError _stub_cacherHrJ)r>modulenamerrbr?rArBrcs  z-load_model.._SafeUnpickler.find_class)rHrIrJ__doc__rfrcrMrArAr?rB_SafeUnpicklersrjdata.pklcs*Gfddd}||||fi|S)Ncs"eZdZZjZjZjZdS)z;load_model.._patched_load..SafePickleModuleN)rHrIrJ UnpicklerloaddumpsloadsrA)rjpicklerArBSafePickleModule*s  rqrA)zip_file map_location pickle_module pickle_filekwargsrq)rj _orig_rebuildrprArB _patched_load(sz!load_model.._patched_loadcpuF)rs weights_onlyi_lang_embed_module state_dict)strictz Loaded state_dict: z missing, z unexpected keysz Missing (first 5): rcsh||d\}}tdr0tdr0jdur0j}|jd|jdkr-|}||}d_||fS)N) audio_signallength_current_lang_ids _lang_embedr)hasattrrshaper)r~rencoded encoded_lenrC lang_bias)modeloriginal_encoder_forwardrArBencoder_forward_with_langEs  z-load_model..encoder_forward_with_langz1 Language conditioning enabled (encoder patched)z%Model loaded and ready for inference.)rk)0nemo.collections.asr collectionsasrprintr rmrjrresolveparentr tokenizerdirospathisabsr]isdirexists write_texttrain_dsmanifest_filepath validation_dsrXdevices acceleratormodelsEncDecHybridRNNTCTCBPEModelzipfileiorprltorch.serialization serialization_loadr7r/ NUM_LANGUAGESrregister_moduleload_state_dictrwencoderrFevalcudatobfloat16)rTrUnemo_asrrWrepo_dirtok_dirdummy_manifestrr_tsrxckpt lang_embedmissing unexpectedrrA)rjrwrrrprB load_models\   &       r c CsHddl}ddl}ddl}g}t|}|jdd}ztd||D]} t| ||} || | } g} tt| dD]/} | d| }tj|dtj d}|d }t j |d | | d }| |||| |q9g}tt| dD]} | d | }t|d }|t|dqstj|tjdd|_tjjdtjd|j| |d}Wdn1swYt|tr|d}t|D]/\} }t|tr|}n t|dr|j}nt|}|| d| | d | | d| |dq| D]}t |q| }t!d|d|d||dddq W|j"|dd|S|j"|ddw)zGTranscribe all samples using NeMo's transcribe API with temp wav files.rNmaya_asr_bench_)prefixraudioarray)dtype sampling_ratesample_z.wavrr"rr)rdevice batch_sizerNr)rrrrz Transcribed /z samples (r.1fz%)T) ignore_errors)#r tempfileshutilrwmkdtemprhrxnprfloat32rrr]writerLANG_NAME_TO_CODEr LANG_TO_IDr7tensorlongrampautocastr transcriber^tuplerrjrrNunlinkrrmtree)rdatasetrsfrrresultstotaltmpdir batch_start batch_endbatch wav_pathsrlrarrsrwav_pathrC lang_name lang_codetranscriptionsrrNpelapsed_samplesrArArBtranscribe_datasetZsb              (5rcCstjdd}|jdtddd|jdtddd|jd td d |jd td d |jdtdd |jdtdd |}t|j|j|j }|j dddt |j |j }tdddlm}m}|ddtjddd}|d|ddd}tdt|dtt|dd t|d!}td"|d#d$|d%d&d'|d(}|rtd)|t| } t| } Wdn1swY| d*} | d+} tdt| d,| d#d-nMtd.t} t|||jd/} t| } td0| d#d1| |d2d3t|d4} tj | | d5| d6d7Wdn 1swYtd8|td9t!| }d:}zt"j#$d}Wn t%yGYnw|j |j |jd;d|jt&| d<t&|d<t&| |d=t'(t)j*+d>|d?d@t,t-dAdBd:dC|dD<|dE}tdFdGtdH|dId&dJ|dKd&dL|dMd&dN|dOd&dP tdGtdFdQdRdSdTdUdSdVdWdSdXdYdSdZd[dSd\dW td]t.d^d_|DD]-}||}t|dRdS|d`dadb|dIdcdb|dKdddb|dMdedb|dOdcdP qtdft/| }tdgt0| }t|dhd4} tj || d)rdecodez Loaded z samples across rz languagesdurationz Total audio: rzs (iz.2fzh)z_predictions_cache.jsonz! Loading cached predictions from rinference_timez) cached predictions (original inference: zs)z Starting transcription...rz Transcription complete in zs (RTF: z.4f)r )rrF) ensure_asciiz Cached predictions to z Computing metrics...unknownzmaya-asr-hybrid-tdt-1.1br rz%Y-%m-%dT%H:%M:%SZnemov1jiwer __version__) checkpointcheckpoint_namemodel_id model_typerrinference_time_sectotal_audio_secrtf timestamprV frameworknormalization_version jiwer_version__meta__r zP================================================================================zOVERALL: wer_norm=rz% space_norm=rz% mer=rz % cer_norm=r%Languagez<15rWzWER Rawz>8zWER Normz>9 SpaceNormz>10MERz>7zCER NormzA-----------------------------------------------------------------css|] }|ds|VqdS)rrNrrrArArBrszmain..rz>7.2fz% z>8.2fz>9.2fz>6.2fz Building sample analysis...zBuilding error analysis...z metrics.json)indentrzsample_analysis.jsonzerror_analysis.jsonz Results written to rz - metrics.jsonz - sample_analysis.jsonz - error_analysis.json)1argparseArgumentParser add_argumentrjrK parse_argsr output_dirrrmkdirrconfigrrdatasetsrrrenvironr cast_columnrwryrropenjsonrmtimerrdumprr7rget_device_namerrrnowrutcstrftimegetattr __import__rrrS)parserargsout_dirrrrdsrpredictions_cachefcacherrt0metricsgpu_nameorrtsample_analysiserror_analysisrArArBmains  &        8P r/__main__)r)9rirrrrfsysrrPrrrrrpathlibrnumpyrr7rrrrr omegaconfr rrrrwrr8Moduler/compilerZrarbrcr_r\rgrjrTrermrnrrrr_rrrSrrr/rHrArArArBs`            ()f= nF s