o
    }oi?                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlZ	d dl
Z
d dlZd dlmZmZ d dlmZmZ G dd dZG d	d
 d
ZG dd dZdS )    N)contextmanager)Path)mock)get_full_pathis_tarred_dataset)flattenmask_sequence_tensorc                   @   s   e Zd Zejjdd ZdS )TestListUtilsc                 C   s   g }| g dg dd | dddggg dd | ddgdggggg dd | dddgdggd	ggg d
d | dddggg dd t|D ]\}}t|d |d kshJ d| dqRdS )zRTest flattening an iterable with different values: str, bool, int, float, complex.)aabbcc)inputgoldenr
   r   r               )r
   r   r   r   r   T      @       @      ?)Tr   r   r   r   z
Test case z failed!N)append	enumerater   )self
test_casesn	test_case r   W/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/common/test_utils.pytest_flatten   s   "&&zTestListUtils.test_flattenN)__name__
__module____qualname__pytestmarkunitr   r   r   r   r   r	      s    r	   c                   @   s4   e Zd Zejjejdg ddefddZdS )TestMaskSequenceTensorndim)r   r   r      c              
   C   sR  d}d}d}t |D ]}tjjd|d}tjjd|d}|dkr3|fttdd|d f |f }n||f}t|}	tjd|d |fd}
|dkrt|	|
d	}t|
D ]4\}}t	||d
d|f |	|d
d|f stJ d| t
||d
|df dksJ d| qUq
tt t|	|
d	 W d   n1 sw   Y  q
dS )z3Test masking a tensor based on the provided length.   
      r   lowhighr   )r+   r,   sizer   )tensorlengths.NzFailed for example g        )rangenprandomrandinttupletorchrandnr   r   equalallr!   raises
ValueError)r   r%   num_examplesmax_batch_sizemax_max_lenr   
batch_sizemax_lentensor_shaper.   r/   masked_tensorblr   r   r   test_mask_sequence_tensor,   s,   &
6,z0TestMaskSequenceTensor.test_mask_sequence_tensorN)	r   r   r    r!   r"   r#   parametrizeintrD   r   r   r   r   r$   +   s    r$   c                   @   s|   e Zd Zejjdd Zejjdd Zejjdd Zejjdd Z	ejjd	d
 Z
ejjdd Zejjdd ZdS )TestPreprocessingUtilsc                    s>  d}dd t |D } fdd|D } }tj|d}tdd }t |D ]}t|| |d|| ks7J t|| |d	|| ksEJ q't||d|ksPJ t||d	|ksZJ ||? t |D ]}t|| |d|| kssJ t|| |d	|| ksJ qct||d|ksJ t||d	|ksJ W d
   n1 sw   Y  t |D ]}t|| |d|| ksJ t|| |d	|| ksJ qt||d|ksJ t||d	|ksJ ||B t |D ]}t|| |d|| ksJ t|| |d	|| ksJ qt||d|ksJ t||d	|ksJ W d
   n	1 s&w   Y  tjtjt}	dd t |D }
||
R t |D ].}tj	|
| dksVJ t|
| |d|
| kseJ t|
| |	d	|
| kstJ qGt|
|d|
ksJ t|
|	d	|
ksJ W d
   d
S 1 sw   Y  d
S )zTest with local pathsr(   c                 S      g | ]}d | dqS file_.testr   .0r   r   r   r   
<listcomp>S       zCTestPreprocessingUtils.test_get_full_path_local.<locals>.<listcomp>c                       g | ]	}t j |qS r   ospathjoinrM   
a_file_reltmpdirr   r   rN   T       manifest.jsonc                 s   8    | D ]}t |  qd V  | D ]}t |  qd S Nr   touchunlinkpathsa_filer   r   r   create_filesZ      zETestPreprocessingUtils.test_get_full_path_local.<locals>.create_filesmanifest_filedata_dirNc                 S   rH   )z./file_rK   r   rL   r   r   r   rN      rO   T)
r0   rR   rS   rT   r   r   dirnameabspath__file__isfile)r   rX   	num_filesaudio_files_relative_pathaudio_files_absolute_pathrh   rf   rc   r   curr_diraudio_files_relative_path_currr   rW   r   test_get_full_path_localM   sn   


 



$z/TestPreprocessingUtils.test_get_full_path_localc              	      s.  d}dd t |D fddD d}tj|d tdd } fd	d
}td|\ |? t |D ]}t|  d| ksKJ t| |d| ksYJ q;t dksdJ t|dksnJ W d   n1 sxw   Y  W d   dS W d   dS 1 sw   Y  dS )Test with paths on AIStore.r(   c                 S   rH   rI   r   rL   r   r   r   rN      rO   zATestPreprocessingUtils.test_get_full_path_ais.<locals>.<listcomp>c                    rP   r   rQ   rU   rW   r   r   rN      rY   
ais://testrZ   c                 s   r[   r\   r]   r`   r   r   r   rc      rd   zCTestPreprocessingUtils.test_get_full_path_ais.<locals>.create_filesc                    s@   t jj| t j d}|v r|}| S td|  )N)startzUnexpected path )rR   rS   relpathri   indexr:   )rS   rel_pathidx)ais_manifest_fileaudio_files_cache_pathrn   r   r   &datastore_path_to_cache_path_in_tmpdir   s
   
z]TestPreprocessingUtils.test_get_full_path_ais.<locals>.datastore_path_to_cache_path_in_tmpdirzInemo.collections.common.parts.preprocessing.manifest.get_datastore_objectre   rg   N)r0   rR   rS   rT   r   r   patchr   )r   rX   rm   ais_data_dirrc   r|   r   r   )rz   r{   rn   rX   r   test_get_full_path_ais   s0   

	
"z-TestPreprocessingUtils.test_get_full_path_aisc                    s   d}dd t |D }d tj d} fdd|D }t |D ] }t|| |dd|| ks2J t||  dd	|| ksAJ q!t||dd|ksMJ t| dd	|ksXJ d
S )rs   r(   c                 S   rH   rI   r   rL   r   r   r   rN      rO   zJTestPreprocessingUtils.test_get_full_path_ais_no_cache.<locals>.<listcomp>rt   rZ   c                    rP   r   rQ   )rM   rx   r~   r   r   rN      rY   F)rf   force_cache)rh   r   N)r0   rR   rS   rT   r   )r   rm   rn   rz   ro   r   r   r   r   test_get_full_path_ais_no_cache   s$   
z6TestPreprocessingUtils.test_get_full_path_ais_no_cachec           	         s   d}t tjtj tj tj   fdd}dD ]L}t|D ]E}tj	j
|dd}tt||}t||d|ksCJ d| d	| d
tjd|}tj|}t||d|kscJ d| d	| d
qqdS )zTest with audio_file_len_limit.
        Currently, get_full_path will always return the input path when the length
        is over audio_file_len_limit, independend of whether the file exists.
        r(   c                    s   d tjj | dS )N )r-   )rT   r1   r2   choice)r   
rand_charsr   r   <lambda>  s    zPTestPreprocessingUtils.test_get_full_path_audio_file_len_limit.<locals>.<lambda>)   i,  i^  r*   )audio_file_len_limitzLimit z: expected z to be returned.~N)liststringascii_uppercaseascii_lowercasedigitsrR   sepr0   r1   r2   r3   strr   r   rS   rT   
expanduser)	r   r;   	rand_namer   r   path_lengthaudio_file_pathaudio_file_path_with_user"audio_file_path_with_user_expectedr   r   r   'test_get_full_path_audio_file_len_limit  s&   
z>TestPreprocessingUtils.test_get_full_path_audio_file_len_limitc                 C   s   t jtdd td W d   n1 sw   Y  t jtdd td W d   n1 s1w   Y  t jtdd tdddd W d   n1 sPw   Y  t jtdd tg d W d   dS 1 snw   Y  dS )	zUMake sure exceptions are raised when audio_file is not a string or a list of strings.zUnexpected audio_file typematchr   N)arB   cr   r   )r   r   r   r!   r9   r:   r   r   r   r   r   test_get_full_path_invalid_type.  s   

"z6TestPreprocessingUtils.test_get_full_path_invalid_typec                 C   sx   t jtdd td W d   n1 sw   Y  t jtdd tdddd W d   dS 1 s5w   Y  dS )	zMake sure exceptions are raised when audio_file is a relative path and
        manifest is not provided or both manifest and data dir are provided simultaneously.
        z$Use either manifest_file or data_dirr   zrelative/pathNzDParameters manifest_file and data_dir cannot be used simultaneously.z/manifest_dir/file.jsonz	/data/dir)rf   rh   r   r   r   r   r   (test_get_full_path_invalid_relative_path>  s   
"z?TestPreprocessingUtils.test_get_full_path_invalid_relative_pathc                 C   s   t ddsJ t ddsJ t ddrJ t ddrJ t ddr#J t ddr*J t ddr1J t ddr8J t ddr?J t ddrFJ t dd rMJ d S )	Nz_file_1.wavztarred_audio_manifest.jsonz#./sharded_manifests/manifest_1.jsonz./file_1.wavzaudio_manifest.jsonz&./sharded_manifests/manifest_test.jsonz
file_1.wavz/data/file_1.wav)r   r   r   r   r   test_is_tarred_datasetK  s   z-TestPreprocessingUtils.test_is_tarred_datasetN)r   r   r    r!   r"   r#   rr   r   r   r   r   r   r   r   r   r   r   rG   L   s    
i
7
#


rG   )rR   r   
contextlibr   pathlibr   unittestr   numpyr1   r!   r5   4nemo.collections.common.parts.preprocessing.manifestr   r   #nemo.collections.common.parts.utilsr   r   r	   r$   rG   r   r   r   r   <module>   s   !