o
    }oi.                     @   s   d dl Z d dlZd dlZd dlmZ d dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ dddddZdZdede Zej dddd Z!ej dddd Z"ej#j$ej dddd Z%G dd dZ&dS )    N)Path)Trainer)	OmegaConf)ASRModelEncDecCTCModelBPEEncDecRNNTBPEModel)CTCDecodingConfig)GreedyCTCInferConfig)RNNTDecodingConfig)GreedyBatchedRNNTInferConfig)run_confidence_benchmark)ConfidenceConfiggףp=
?g=
ףp=?gQ?g)\(?))tokenctc)r   rnnt)wordr   )r   r         
   module)scopec                  C   *   t jdd} | tddd |  } | S )Nstt_en_conformer_ctc_small
model_namer   cpudevicesaccelerator)r   from_pretrainedset_trainerr   evalmodel r$   h/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/asr/confidence/test_asr_confidence.pyconformer_ctc_bpe_model(      r&   c                  C   r   )N!stt_en_conformer_transducer_smallr   r   r   r   )r   r   r    r   r!   r"   r$   r$   r%   conformer_rnnt_bpe_model0   r'   r)   c                 C   s   g }g }t | t d }t|d/}|D ]!}t|}t |d dd}|t|  ||d  qW d    ||fS 1 sCw   Y  ||fS )Nzasr/an4_val.jsonraudio_filepathz/data/z/.data/text)r   openjsonloadsreplaceappendstrabsolute)test_data_dir	filepathsreference_textsmanifestflineitem
audio_filer$   r$   r%   audio_and_texts8   s   

r<   c                   @   sv   e Zd Zejjejjejjejddejdddd Z	ejjejjejjejdddd Z
d	S )
TestASRConfidenceBenchmarkr   )r   r   target_level)r   r   c                 C   s   |dkr|n|}t |tsJ |\}}|dkrtddntdd}	||dkr.tdd|	d	nt|	d
 t 0}
tj	t
t|||||
d| t
dddtj t||f dddgtds`J W d    d S 1 skw   Y  d S )Nr   r   T)preserve_token_confidence)preserve_word_confidencer   greedy_batchfused_batch_sizestrategyconfidence_cfgrF   )plot_dir      ?g      ?g        )atol)
isinstancer   r   change_decoding_strategyr
   r   tempfileTemporaryDirectorynpallclosearrayr   mathinf
ECE_VALUESTOL)selfr   r>   r<   r&   r)   r#   r5   r6   rF   tmpdirr$   r$   r%   test_run_confidence_benchmarkK   s.   

$"z8TestASRConfidenceBenchmark.test_run_confidence_benchmarkc           	   	   C   s   t  jjdksJ d|dkr|n|}t|tsJ tddi}d|i}d|i}t ddd	i|}||d
kr?tdd|dnt	|d |j
jjjjdksPJ ||d
krdtddtddd	i|dnt	tddd	i|d |j
jjjjdks|J d S )NgQ?z&default `alpha` is supposed to be 0.33r   temperaturerI   
method_cfgconfidence_method_cfgr@   Tr   rA   greedyrC   rG   preserve_frame_confidence)rD   rE   r\   )r\   r$   )r   rZ   alpharK   r   r   createrL   r
   r   cfgdecodingrF   r   r	   r\   r[   )	rV   r   r&   r)   r#   conftest_args_maintest_args_greedyrF   r$   r$   r%   test_deprecated_config_argsk   s.   	z6TestASRConfidenceBenchmark.test_deprecated_config_argsN)__name__
__module____qualname__pytestmarkpleasefixmeintegrationwith_downloadsparametrizerX   re   r$   r$   r$   r%   r=   J   s    r=   )'r.   rR   rM   pathlibr   numpyrO   ri   lightning.pytorchr   	omegaconfr   nemo.collections.asr.modelsr   r   r   2nemo.collections.asr.parts.submodules.ctc_decodingr   9nemo.collections.asr.parts.submodules.ctc_greedy_decodingr	   3nemo.collections.asr.parts.submodules.rnnt_decodingr
   :nemo.collections.asr.parts.submodules.rnnt_greedy_decodingr   Bnemo.collections.asr.parts.utils.asr_confidence_benchmarking_utilsr   5nemo.collections.asr.parts.utils.asr_confidence_utilsr   rT   
TOL_DEGREEpowrU   fixturer&   r)   rj   rm   r<   r=   r$   r$   r$   r%   <module>   s4   




