o
    Si5                     @   s$  d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
mZmZ d dlmZ d dlmZmZ d dlmZmZ d d	lmZmZmZ d d
lmZmZ 			d"dededee defddZ			d#dedee dedede
ee
eeeef f f f
ddZ dd Z!dd Z"d d! Z#dS )$    N)defaultdict)ProcessPoolExecutor)Path)DictOptionalUnion)tqdm)fix_manifests$validate_recordings_and_supervisions)	RecordingRecordingSet)AlignmentItemSupervisionSegmentSupervisionSet)Pathlikeresumable_download.F!https://data.deepai.org/timit.zip
target_dirforce_downloadbase_urlreturnc           	      C   s   t | } | jddd d}| | }|d}|d }| r,td| d| d |S t|||d	 t|!}|jddd t	|
 d
D ]
}||t| qGW d   n1 s\w   Y  |  |  |S )a_  
    Download and unzip the dataset TIMIT.
    :param target_dir: Pathlike, the path of the dir to store the dataset.
    :param force_download: bool, if True, download the zips no matter if the zips exists.
    :param base_url: str, the URL of the TIMIT dataset to download.
    :return: the path to downloaded and extracted directory with data.
    Tparentsexist_okz	timit.zip z
.completedz	Skipping z	 because z exists.)filenamer   zExtracting filesN)r   mkdirwith_suffixis_filelogginginfor   zipfileZipFiler   namelistextractstrunlinktouch)	r   r   r   zip_namezip_path
corpus_dircompleted_detectorzip_filenames r/   H/home/ubuntu/.local/lib/python3.10/site-packages/lhotse/recipes/timit.pydownload_timit   s&   
r1   0      r+   
output_dir
num_phonesnum_jobsc                    s  t | } |  sJ d|  |durt |}|jddd tt}g d}i }|dv r1t|}ntdt \ |D ]"}g }|dkrStd	 t		t
| d
 }n-|dkrlt		t
| d }tt fdd|}nt		t
| d }ttfdd|}t| d g }	g }
|dkrt|d| dD ].}zt|||\}}|	| |
| W q ty } zt|j W Y d}~qd}~ww n^t|Q}g }|D ]}||t||| qt|d| dD ]-}z| \}}|	| |
| W q ty } zt|j W Y d}~qd}~ww W d   n	1 s"w   Y  t|	}t|
}t||\}}t|| |durX||d| d  ||d| d  ||d||< q<|S )a,  
    Returns the manifests which consists of the Recodings and Supervisions.
    :param corpus_dir: Pathlike, the path of the data dir.
    :param output_dir: Pathlike, the path where to write and save the manifests.
    :param supervision_lvl: str='phone', the level of the supervision, 'phone', 'word' or 'text'.
    :param num_phones: int=48, the number of phones (60, 48 or 39) for modeling and 48 is regarded as the default value.
    :return: a Dict whose key is the dataset part, and the value is Dicts with the keys 'audio' and 'supervisions'.
    zNo such directory: NTr   )TRAINDEVTEST)<   r2   '   z0The value of num_phones must be in [60, 48, 39].r7   zstarting....z/data/TRAIN/*/*/*.WAVr8   z/data/TEST/*/*/*.WAVc                       |  dd   v S N/splitlowerx)dev_spksr/   r0   <lambda>h       zprepare_timit.<locals>.<lambda>c                    r<   r=   r@   rC   )	test_spksr/   r0   rF   o   rG   z dataset manifest generation.r3   z
Preparing z	 manifesttimit_supervisions_z	.jsonl.gztimit_recordings_)
recordingssupervisions)r   is_dirr   r   dictget_phonemes
ValueErrorget_speakersprintglobr&   listfilterr    debugr   prepare_recordingappendFileNotFoundErrorwarningstrerrorr   submitresultr   from_recordingsr   from_segmentsr	   r
   to_file)r+   r4   r5   r6   	manifestsdataset_partsphones_dictpart	wav_filesrK   rL   wav_file	recordingsupervisioneexresultsrrecording_setsupervision_setr/   )rE   rH   r0   prepare_timit:   s   










ro   c                 C   s$  t |  d}|d d |d d d  }|d }t| d}t| d}t| d}tj| |d	}	t|d
}
d|
	 
dddd  }W d    n1 sXw   Y  g }t|d
4}
|
 }|D ]%}| d\}}}t||	j }t||	j }|t||||  qkW d    n1 sw   Y  g }t|d
<}
|
 }|D ]-}| d\}}}t||	j }t||	j }|dkr|| }|t||||  qW d    n1 sw   Y  t||d|	jdd|| drdnd| d	}|d|d|}|	|fS )Nr>   r?   -z.TXTz.WRDz.PHN)pathrecording_idrl    
   r:   g        r   Englishmmalefemale)	idrt   startdurationchannellanguagespeakergendertextwordphone)r&   striprA   r   r   r   	from_fileopenjoinreadrstrip	readlinesfloatsampling_raterX   r   r   r~   rB   
startswithwith_alignment)rf   r5   rc   itemsidxr   	text_file	word_file
phone_filerg   fr   word_alignmentslineslinestetr   r}   endphone_alignmentsr   segmentr/   r/   r0   rW      s^   $	rW   c                 C   s*  i }| t dkrtd d|d< d|d< d|d< d|d< d|d< d|d< d	|d	< d	|d
< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d |d < d|d!< d|d"< d#|d#< d#|d$< d%|d%< d&|d&< d'|d'< d(|d(< d)|d)< d*|d+< d,|d,< d|d< d-|d-< d|d< d-|d.< d/|d/< d0|d0< d1|d1< d|d2< d*|d3< d4|d5< d6|d6< d7|d7< d8|d8< d9|d9< d*|d:< d;|d;< d<|d<< d=|d=< d=|d>< d?|d?< d@|d@< dA|dA< dB|dB< dC|dC< |S | t dDkrtdE d|d< d|d< d|d< d|d< d|d< d|d< d|d	< d|d
< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d|d< d,|d< d|d< d-|d< d|d< d|d< d|d< d|d< d|d< d |d < d|d!< d|d"< d#|d#< d#|d$< d%|d%< d%|d&< d'|d'< d(|d(< d)|d)< d|d+< d,|d,< d|d< d|d< d-|d-< d-|d.< d/|d/< d0|d0< d1|d1< d|d2< d|d3< d4|d5< d6|d6< d7|d7< d8|d8< d9|d9< d|d:< d;|d;< d<|d<< d=|d=< d=|d>< d?|d?< d@|d@< dA|dA< dB|dB< d8|dC< |S tdF |S )Gzo
    Choose and convert the phones for modeling.
    :param num_phones: the number of phones for modeling.
    r2   zUsing 48 phones for modeling!silaaaeahaoawaxzax-heraxraybvclbclchddcldhdxehelry   emenngengepieyr   ggclzh#hhhvihixiyjhkclkcllnnxowoyppaupclr   qrl   sshttclthuhuwuxvwyzzhr;   zUsing 39 phones for modeling!zUsing 60 phones for modeling!)intr    rV   )r5   phonemesr/   r/   r0   rO      s
  
G

rO   c                  C   s   g d} g d}|| fS )N)fdhc0felc0fjlm0fmgd0fmld0fnlp0fpas0fpkt0mbpm0mcmj0mdab0mgrt0mjdh0mjln0mjmp0mklt0mlll0mlnt0mnjm0mpam0mtas1mtls0mwbt0mwew0)2fadg0faks0fcal1fcmh0fdac1fdms0fdrw0fedw0fgjd0fjem0fjmg0fjsj0fkms0fmah0fmml0fnmr0frew0fsem0majc0mbdg0mbns0mbwm0mcsh0mdlf0mdls0mdvc0mers0mgjf0mglb0mgwt0mjar0mjfc0mjsw0mmdb1mmdm2mmjr0mmwh0mpdf0mrcs0mreb0mrjm4mrjr0mroa0mrtk0mrws1mtaa0mtdt0mteb0mthc0mwjg0r/   )test_spkdev_spkr/   r/   r0   rQ   i  s   5rQ   )r   Fr   )Nr2   r3   )$rS   r    r"   collectionsr   concurrent.futuresr   pathlibr   typingr   r   r   	tqdm.autor   lhotser	   r
   lhotse.audior   r   lhotse.supervisionr   r   r   lhotse.utilsr   r   boolr&   r1   r   ro   rW   rO   rQ   r/   r/   r/   r0   <module>   sT   
&
h6 