o
    }oiFc                     @   s   d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlZd dlZd dlmZmZmZmZ d dlmZmZmZmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ dee  defddZ!G dd dZ"dS )    N)deepcopy)List)Mockpatch)WERword_error_rateword_error_rate_detailword_error_rate_per_utt)CTCBPEDecodingCTCBPEDecodingConfigCTCDecodingCTCDecodingConfig)RNNTBPEDecodingRNNTDecoding)
Hypothesis)CharTokenizer) assert_dataclass_signature_match
vocabularyreturnc              	   C   sf   t dttddd | D d td}W d    n1 s"w   Y  t|dt|jd |S )	Nzpathlib.Path.open
c                 S   s   g | ]}t |qS  )repr).0charr   r   Z/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/asr/test_asr_metrics.py
<listcomp>'       z8build_char_tokenizer_with_vocabulary.<locals>.<listcomp>return_valuea_path_which_will_not_be_used	tokenizer)
vocab_size)r   r   ioStringIOjoinr   setattrr!   )r   char_tokenizerr   r   r   $build_char_tokenizer_with_vocabulary&   s
   (
r'   c                	   @   sD  e Zd Zdgeej dg ZeeZd5de	de
de
dejfdd	Zde	de
dejfd
dZde	de	de
fddZejjdd Zejjejdddgejdddgdd Zejjejdddgdd Zejjejdddgdd Zejjejdddgejdddgdd Zejjejdddgejdddgdd  Zde	de	de
fd!d"Zd#d$ Zde	de	dede
fd%d&Zejjejdddgejdddgd'd( Zejjejdddgd)d* Zejjd+d, Z ejjd-d. Z!ejjd/d0 Z"ejjd1d. Z!ejjd2d3 Z#d4S )6TestWordErrorRate 'Ftxtuse_tokenizeras_logprobsr   c                    s   |rj j}j |}ntj}tfddttjD   fdd|D }g }d}|D ]}||kr=|| n
|| || |}q1t	|
d}	|sV|	S |	tj}	tjjj|	d |d}
|

d}
|
S )Nc                       g | ]	} j | |fqS r   r   r   iselfr   r   r   :       z<TestWordErrorRate.__string_to_ctc_tensor.<locals>.<listcomp>c                       g | ]} | qS r   r   r   cchar_to_indr   r   r   ;   r   r   )num_classes)r&   r!   text_to_idslenr   dictrangeappendtorchTensor	unsqueezetoint64nn
functionalone_hot)r3   r+   r,   r-   blank_idstring_in_id_formctc_listprev_idr7   tensor
new_tensorr   r9   r3   r   __string_to_ctc_tensor3   s*   
 


z(TestWordErrorRate.__string_to_ctc_tensorc                    sT   |r	j |}ntfddttjD   fdd|D }t|dS )Nc                    r.   r   r/   r0   r2   r   r   r   U   r4   zBTestWordErrorRate.__reference_string_to_tensor.<locals>.<listcomp>c                    r5   r   r   r6   r8   r   r   r   V   r   r   )	r&   r<   r>   r?   r=   r   rA   rB   rC   )r3   r+   r,   rJ   r   rO   r   __reference_string_to_tensorP   s
    z.TestWordErrorRate.__reference_string_to_tensor
prediction	referencec           	      C   sx   |  ||}| ||}|jdkr|dd |dd ||d |tt|gd | \}}}| 	 }|
 S )Nr      predictionspredictions_lengthstargetstargets_lengths)(_TestWordErrorRate__string_to_ctc_tensor._TestWordErrorRate__reference_string_to_tensorbatch_dim_index
transpose_rA   rM   r=   computedetachcpuitem)	r3   werrR   rS   r,   predictions_tensortargets_tensorres_r   r   r   get_werY   s   
zTestWordErrorRate.get_werc                 C   sj  t dgdgddksJ t dgdgddksJ t dgdgddks$J t dgd	gddks0J t dgd
gddks<J t dgdgddksHJ tdgdgdd dksVJ tdgdgdd dksdJ tdgdgdd dksrJ tdgd	gdd dksJ tdgd
gdd dksJ tdgdgdd dksJ tdgdgdtddtdtdtdfksJ tddgddgddksJ tdgdgddksJ tdgdgddksJ tdgd
gdddksJ tdgdgddgdfksJ tddgddgdtddgdfksJ td
dgddgdddgdfksJ td
dgddgddddgdfks3J d S )Ncatcot
hypotheses
references      ?GPUzG P U      @ducati motorcycle
motorcycleducuti motorcycle      ?za B ca b cUUUUUU?r    infgpu)       @rT   rm   rm           )rm   rT   rz   rz   rm   )ro   rT   ry   rz   rm   T)rk   rl   use_cer)rm      rz   rm   rz   katry   gUUUUUU?g?gUUUUUU?g333333?)r   r   floatr	   r2   r   r   r   test_wer_functionj   sF   
*


z#TestWordErrorRate.test_wer_functionr\   r   rT   test_wer_bpeTc                 C   s   |  dd|dksJ |  dd|dksJ |  dd|dks!J |  dd|dks,J |  dd	|d
ks7J t|  dd|d dk sFJ d S Nrh   ri   rm   rx   zg p uro   rp   rq   rr   rs   za f crt   ru   ư>)get_wer_ctcabsr3   r\   r   r   r   r   test_wer_metric_simple   s   "z(TestWordErrorRate.test_wer_metric_simplec                    sz    fdd}t dD ]0}tdd}tdd}||}||}| r:t j|||dt|g|gd dk s:J q
d	S )
<This test relies on correctness of word_error_rate function.c                       d  fddt| D S )Nrv   c                 3   "    | ]}t d  jV  qdS rv   Nrandomchoicer$   r   r   rf   r2   r   r   	<genexpr>        zXTestWordErrorRate.test_wer_metric_randomized.<locals>.__random_string.<locals>.<genexpr>r$   r?   lengthr2   r   r   __random_string      zETestWordErrorRate.test_wer_metric_randomized.<locals>.__random_string   rT      )rR   rS   r   rj   r   N)r?   r   randintstripr   r   r   r3   r   !_TestWordErrorRate__random_stringtest_idn1n2s1s2r   r2   r   test_wer_metric_randomized   s    z,TestWordErrorRate.test_wer_metric_randomizedc                 C   s   ddi}|rt || j}t|dd}nt|| j }t|dd}| jd|dd   	 }|g dks8J |j
|}|g d	ksFJ |j
|}|dksRJ d S )
NstrategygreedyFr{   rh   )r,   r      rT      )r7   at)r
   r&   r   r   r   copyrZ   intnumpytolistdecodingdecode_ids_to_tokensdecode_tokens_to_str)r3   r   decoding_configr   rb   tokenstokens_decodedstr_decodedr   r   r   test_wer_metric_decode   s   z(TestWordErrorRate.test_wer_metric_decodec           	      C   P  d|d}t t|| jdd}| jd|dd }|dkr#|dd	 d|j_|jj|dd
}|d }t	|t
s:J |dkrB|d n
|d d dd d f }|jtg d  dks]J |jdksdJ |jdkskJ |jd |k svJ |jdks}J tj|jd	|  gtjd}|jj||dd}|d }t	|t
sJ |jdksJ d S Nr   )r   r\   Fr   rh   T)r-   r   rT   return_hypothesesr   r   dtype)decoder_lengthsr   )r   r   r   rZ   r   r]   r   preserve_alignmentsctc_decoder_predictions_tensor
isinstancer   
y_sequencerA   rM   sumscoretext
alignmentsallr   shapelong	r3   r\   r   r   rb   rM   hypsampler   r   r   r   !test_wer_metric_return_hypothesis   (   
& z3TestWordErrorRate.test_wer_metric_return_hypothesisc           	      C   r   r   )r   r
   r&   rZ   r   r]   r   r   r   r   r   r   rA   rM   r   r   r   r   r   r   r   r   r   r   r   r   )test_wer_metric_subword_return_hypothesis   r   z;TestWordErrorRate.test_wer_metric_subword_return_hypothesisc           
      C   s   t tdg |dgd}|r%t | jjjt| j|| jjtd}t|dd}nt t	| j
| j
 || jtd}t|dd}| ||}|d d |tt	|gd | \}}	}	|  }| S )	Nrm   r   r   r   r   )rI   r    r   r   specFr   )rI   
labels_mapr   r   r   rU   )r   r   r&   r    r!   r   ids_to_textr
   r   r=   r   r   (decode_token_to_str_with_vocabulary_mockr   r[   rA   rM   r^   r_   r`   ra   )
r3   rR   rS   r   #ctc_decoder_predictions_tensor_mockr   rb   rd   re   rf   r   r   r   r     s<   zTestWordErrorRate.get_wer_ctcc                    s   d  fdd|D S )Nrv   c                    s   g | ]} j | qS r   r/   )r   id_r2   r   r   r   5  s    zNTestWordErrorRate.decode_token_to_str_with_vocabulary_mock.<locals>.<listcomp>)r$   )r3   idsr   r2   r   r   4  s   z:TestWordErrorRate.decode_token_to_str_with_vocabulary_mockc                 C   s   t tdg |dgd}|r&t | jjjt| j|| jjtd}t||dd}nt t	| j
| j
 || jtd}t||dd}| ||}|jdkrN|dd	 |d d |tt	|gd
 | \}	}
}
|	  }	|	 S )Nrm   r   r   )rI   r    rnnt_decoder_predictions_tensorr   r   F)r\   r{   )rI   r   r   r   r   r   rT   rU   )r   r   r&   r    r!   r   r   r   r   r=   r   r   r   r   r[   r\   r]   rA   rM   r^   r_   r`   ra   )r3   rR   rS   r\   r   $rnnt_decoder_predictions_tensor_mockr   rb   rd   re   rf   r   r   r   get_wer_rnnt7  s@   
zTestWordErrorRate.get_wer_rnntc                 C   s   |  dd||dksJ |  dd||dksJ |  dd||dks$J |  dd||dks0J |  dd	||d
ks<J t|  dd||d dk sLJ d S r   )r   r   r   r   r   r   test_rnnt_wer_metric_simple[  s   $z-TestWordErrorRate.test_rnnt_wer_metric_simplec                    s|    fdd}t dD ]1}tdd}tdd}||}||}| r;t j||d|dt|g|gd d	k s;J q
d
S )r   c                    r   )Nrv   c                 3   r   r   r   r   r2   r   r   r   l  r   z]TestWordErrorRate.test_rnnt_wer_metric_randomized.<locals>.__random_string.<locals>.<genexpr>r   r   r2   r   r   r   k  r   zJTestWordErrorRate.test_rnnt_wer_metric_randomized.<locals>.__random_stringr   rT   r   r   )rR   rS   r\   r   rj   r   N)r?   r   r   r   r   r   r   r   r   r2   r   test_rnnt_wer_metric_randomizedf  s    z1TestWordErrorRate.test_rnnt_wer_metric_randomizedc           	      C   sx  ddt | j}}}td tj|||tjd}tjd||gtjd}||td|dgd < t }t	|| jd}|j
||dd}|d }t|jtjsPJ |jtj|tjdks]J |jd	ksdJ t |jdksmJ |jd u stJ tddd
}t	|| jd}|j
||dd}|d }t|jtjsJ |jtj|tjdksJ |jd	ksJ t |jdksJ |jd usJ d S )NrT      r   r   sizer   r/   Tr   rv   r   compute_timestamps   )r=   r   rA   manual_seedrandnfloat32r   int32r   r   r   r   r   rB   r   rM   r   	timestampr   	r3   BTVdecoder_outputsdecoder_lensdecoding_cfgr   r   r   r   r   test_char_decoding_logprobs}  s.   
z-TestWordErrorRate.test_char_decoding_logprobsc           	      C   v  dd| j j}}}td tj|||tjd}tjd||gtjd}||td|dgd < t }t	|| j d}|j
||dd}|d }t|jtjsOJ |jtj|tjdks\J |jd	kscJ t|jdkslJ |jd u ssJ tddd
}t	|| j d}|j
||dd}|d }t|jtjsJ |jtj|tjdksJ |jd	ksJ t|jdksJ |jd usJ d S NrT   r   r   r   r   r    Tr   rv   r   r   r&   r!   rA   r   r   r   r   r   r   r
   r   r   r   rB   r   rM   r   r=   r   r   r   r   r   r   test_subword_decoding_logprobs  .   
z0TestWordErrorRate.test_subword_decoding_logprobsc           
      C   s  ddt | j}}}td tjd|d ||gtjd}tjd||gtjd}||td|dgd < t }t|| jd}|j	||dd}|d }t
|jtjsTJ |jtj|tjdksaJ |jd	kshJ t |jdksqJ |jd u sxJ tddd
}t|| jd}tt |j	||dd}	W d    n1 sw   Y  tddd
}t|| jd}|j	||dd}|d }t
|jtjsJ |jtj|tjdksJ |jd	ksJ t |jdksJ |jd u sJ d S )NrT   r   r   r   r/   Tr   r   rv   r   Fr   )r=   r   rA   r   r   r   r   r   r   r   r   r   rB   r   rM   r   r   r   pytestraises
ValueError
r3   r   r   r   r   r   r   r   r   rf   r   r   r   test_char_decoding_labels  s8   
z+TestWordErrorRate.test_char_decoding_labelsc           	      C   r   r   r   r   r   r   r   r     r   c           
      C   s  dd| j j}}}td tjd|d ||gtjd}tjd||gtjd}||td|dgd < t }t|| j d}|j	||dd}|d }t
|jtjsSJ |jtj|tjdks`J |jd	ksgJ t|jdkspJ |jd u swJ tddd
}t|| j d}tt |j	||dd}	W d    n1 sw   Y  tddd
}t|| j d}|j	||dd}|d }t
|jtjsJ |jtj|tjdksJ |jd	ksJ t|jdksJ |jd u sJ d S )NrT   r   r   r   r   Tr   r   rv   r   Fr   )r&   r!   rA   r   r   r   r   r   r
   r   r   r   rB   r   rM   r   r=   r   r   r   r   r   r   r   r   r   test_subword_decoding_labels  s8   
z.TestWordErrorRate.test_subword_decoding_labelsN)F)$__name__
__module____qualname__liststringascii_lowercaser   r'   r&   strboolrA   rB   rZ   r[   rg   r   markunitr   parametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r(   .   s\    	
0#$


&
r(   )#dataclassesr"   r   r   r   r   typingr   unittest.mockr   r   r   rA    nemo.collections.asr.metrics.werr   r   r   r	   2nemo.collections.asr.parts.submodules.ctc_decodingr
   r   r   r   3nemo.collections.asr.parts.submodules.rnnt_decodingr   r   +nemo.collections.asr.parts.utils.rnnt_utilsr   "nemo.collections.common.tokenizersr   nemo.utils.config_utilsr   r   r'   r(   r   r   r   r   <module>   s"   