o
     /i.                     @   s  d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m	Z	m
Z
 ddlmZmZ edZedZed	Zed
ZedZedZdedefddZdedefddZdedefddZdedefddZdee defddZdee dee fddZdddddd ddd!d"d#dd#d$d#dd%d&d%dd'd(d)dd*d+d,dd-Z ed.Z!d/ed0efd1d2Z"e#d3krPi Z$e % D ]*\Z&Z'ee&Z(e() sed4e& Z(e() se*d5e& d6 qe"ee(e'e$e'd7 < qe*d8d9  e*d: e*d9  e*d;d<d=d>d?d=d@dAd=dBd?d=dCdA	 e*dD e+e$% dEdF dGD ]+\Z,Z-e-dH Z.e*e,d<d=e.dI dJdKe.dL dMdKe.dN dJdKe.dO dMdP
 q&dS dS )QzO
Compute all 4 metric tiers for API baseline models.
Normalization version: v1
    N)defaultdict)Path)datetimetimezone)wercerz([\u200b-\u200f\u2028-\u202f\ufeff\u00ad]z'[!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]z[\u0964\u0965\u0970\u0971]z)[\u2010-\u2027\u2030-\u205e\u2e00-\u2e4f]z>[\u2018\u2019\u201a\u201b\u201c\u201d\u201e\u201f\u00ab\u00bb]z	\([^)]*\)textreturnc                 C   s   t d| } |  } | S )z*Step for wer_raw: NFC unicode + trim only.NFC)unicodedata	normalizestripr    r   4/home/ubuntu/training/api_results/compute_metrics.pynorm_raw!   s   r   c                 C   s   t d| } td| } td| } d|  } td| } | dddddd} t	d| } t
d| } td| } |  } d|   } | S )	zESteps for wer_norm: NFKC + strip ZW + whitespace + punct + case fold.NFKC  'u   —-u   –u   ‒)r   r   ZW_CHARSsubANNOTATIONSjoinsplitQUOTESreplacePUNCT_COMMONPUNCT_INDICPUNCT_EXTENDEDlowerr   r   r   r   r   norm_standard'   s   r"   c                 C   s   t | ddS )z1Steps for MER: norm_standard + remove ALL spaces.r   r   )r"   r   r   r   r   r   norm_mer>   s   r#   c                 C   sN   t | } tdd| } dD ]\}}tdD ]}| t|| t|} qq| S )z@Steps for wer_numcanon: norm_standard + number canonicalization.z	(\d),(\d)z\1\2)	)if	  0)i	  r$   )if
  r$   )i
  r$   )if  r$   )i  r$   )if  r$   )i  r$   )if  r$   
   )r"   rer   ranger   chrstr)r   offsetbaseir   r   r   norm_numcanonB   s   r-   samplesc           '         s  t t}| D ]}||d  | qi  g g }}g g }}g g }}g g }	}
t| D ] }|| }g g }}g g }}g g }}g g }}d}|D ]}|d }|ddpZd}| sc|d7 }t|t|}}t|t|}}t	|t	|}}t
|t
|}}| sqM|| || r|nd ||r|nd ||r|nd ||r|nd ||r|nd ||r|nd ||r|nd qMt||d } t||d }!t||d }"t||d }#t||d }$t|t| d	t|!d	t|"d	t|$d	t|#d	|t| |! d	t|!|" d	t|!|$ d	d
d |< || || || || || || |	| |
| q.t| tt||d d	tt||d d	tt||d d	tt|	|
d d	tt||d d	d d< dd  D }%t|%}&|&tt fdd|%D |& d	tt fdd|%D |& d	tt fdd|%D |& d	tt fdd|%D |& d	tt fdd|%D |& d	d d<  S )z1Compute all 4 metric tiers for a list of samples.languager   	reference
transcriptr      z<empty>d      )raw_to_normnorm_to_numcanonnorm_to_mer)	n_sampleswer_rawwer_normwer_numcanonmercer_normempty_hypothesesnormalization_delta)r8   r9   r:   r;   r<   r=   __overall__c                 S   s   g | ]	}| d s|qS )_)
startswith).0kr   r   r   
<listcomp>   s    z/compute_metrics_for_samples.<locals>.<listcomp>c                 3       | ]	} | d  V  qdS )r9   Nr   rC   lresultr   r   	<genexpr>       z.compute_metrics_for_samples.<locals>.<genexpr>c                 3   rF   )r:   Nr   rG   rI   r   r   rK      rL   c                 3   rF   )r;   Nr   rG   rI   r   r   rK      rL   c                 3   rF   )r<   Nr   rG   rI   r   r   rK      rL   c                 3   rF   )r=   Nr   rG   rI   r   r   rK      rL   )n_languagesr9   r:   r;   r<   r=   __macro_avg__)r   listappendsortedkeysgetr   r   r"   r-   r#   compute_wercompute_cerlenroundextendsum)'r.   by_langsall_ref_rawall_hyp_rawall_ref_normall_hyp_norm
all_ref_nc
all_hyp_ncall_ref_merall_hyp_merlanglang_samplesrefs_rawhyps_raw	refs_norm	hyps_normrefs_nchyps_ncrefs_merhyps_merempty_countrefhyprrhrrnhnrnchncrmhmw_raww_normw_ncc_normm_er	lang_keysn_langsr   rI   r   compute_metrics_for_samplesQ   s   







 $$&



	r   c                 C   s  g }| D ]}|d }| ddpd}t|}t|}t|}t|}t|}	t|}
g }||kr5|d ||kr>|d ||krK||krK|d | sT|d td|s`td|ri||kri|d	 ||d
 |d |||||||	|
| dd|d q|S )z#Build sample_analysis.json entries.r0   r1   r   exact_matchexact_match_normpunctuation_only_diffempty_hypothesisz\dnumeric_mismatchidr/   detected_lang)r   r/   r0   
hypothesisref_normhyp_normref_numcanonhyp_numcanonref_merhyp_merdetected_languageflags)rS   r"   r-   r#   rP   r   r&   search)r.   outr[   ro   rp   rs   rt   ru   rv   rw   rx   r   r   r   r   build_sample_analysis   sJ   






r   zelevenlabs-scribe-v2zElevenLabs Scribe v2)model_iddisplay_name
model_typezelevenlabs-scribe-v1zElevenLabs Scribe v1zgemini-3-flash-strictzGemini 3 Flash (Strict)zgemini-3-flash-previewzGemini 3 Flash (Preview)zgemini-2.5-flashzGemini 2.5 Flashzgemini-3.1-prozGemini 3.1 Prozgemini-3.1-pro-previewzsarvam-saaras-v3zSarvam Saaras v3z	saaras:v3)zelevenlabs_full_6k.jsonlzelevenlabs_v1_full_6k.jsonlz#gemini_3_flash_strict_full_6k.jsonlz+gemini_gemini_3_flash_preview_full_6k.jsonlzgemini_flash_v2_full_6k.jsonlz+gemini_gemini_3_1_pro_preview_full_6k.jsonlzsarvam_full_6k.jsonlz'/home/ubuntu/training/benchmark_outputsfilenameinfoc           
      C   s  |d }t dd  t d|d  d|  d t d  t| }dd	 |D }W d    n1 s3w   Y  t d
t| d t|}tdd |D }d| d||d dddt|ddttj	
dddd|d< |d }t d|d dd|d dd |d! dd"|d# d t|}t| d }	|	jd$d$d% t|	d& d'}tj||dd(d) W d    n1 sw   Y  t|	d* d'}tj||dd(d) W d    n1 sw   Y  t d+|	 d, |S )-Nr   
z<============================================================zProcessing: r   z ()c                 S   s   g | ]}t |qS r   )jsonloads)rC   liner   r   r   rE     s    z!process_model.<locals>.<listcomp>z	  Loaded z samplesc                 s   s    | ]	}| d dV  qdS )durationr   N)rS   )rC   r[   r   r   r   rK     rL   z process_model.<locals>.<genexpr>zapi/baseliner   z"BayAreaBoys/indic-asr-benchmark-6kr2   r   r4   z%Y-%m-%dT%H:%M:%SZv1api)
checkpointcheckpoint_namer   r   dataset
batch_sizeinference_time_sectotal_audio_secrtf	timestampnormalization_version	framework__meta__r@   z  Overall: wer_raw=r9   z.2fz  wer_norm=r:   z  mer=r<   z  cer_norm=r=   T)parentsexist_okzmetrics.jsonwF)indentensure_asciizsample_analysis.jsonz  Written to /)printopenrV   r   rY   rW   r   nowr   utcstrftimer   OUTPUT_BASEmkdirr   dump)
r   r   r   fr.   metricstotal_audioosample_analysisout_dirr   r   r   process_model  sH   


8r   __main__z!/home/ubuntu/training/api_resultszSKIP: z
 not foundr   z

zP================================================================================zCOMPARISON TABLE (all models)Modelz<30r   zWER Rawz>8zWER Normz>9zWER NumzCER NormzF----------------------------------------------------------------------c                 C   s   | d d d S )Nr2   r@   r:   r   )xr   r   r   <lambda>I  s    r   )keyr@   r9   z>7.2fz% r:   z>8.2fr;   r=   %)/__doc__r   r&   r   syscollectionsr   pathlibr   r   r   jiwerr   rT   r   rU   compiler   r   r   r    r   r   r)   r   r"   r#   r-   rO   dictr   r   MODELSr   r   __name__resultsitemsr   r   fpathexistsr   rQ   midmr   r   r   r   r   <module>   s    





]0&
4
.D