o
    Siu                     @   s  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZmZmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZmZmZ ddlm Z m!Z!m"Z" dZ#dZ$dZ%	dde de&fddZ'		dde dee  de&fddZ(dd Z)dS )a  
VoxConverse is an audio-visual diarisation dataset consisting of multispeaker clips of human speech, extracted from YouTube videos.
Updates and additional information about the dataset can be found at our website (https://www.robots.ox.ac.uk/~vgg/data/voxconverse/index.html).
    N)defaultdict)ProcessPoolExecutor)Path)DictIterableOptionalUnion)	HTTPError)tqdm)fix_manifests$validate_recordings_and_supervisions)	RecordingRecordingSet)AlignmentItemSupervisionSegmentSupervisionSet)Pathlikeis_module_availableresumable_downloadzNhttps://www.robots.ox.ac.uk/~vgg/data/voxconverse/data/voxconverse_dev_wav.zipzOhttps://www.robots.ox.ac.uk/~vgg/data/voxconverse/data/voxconverse_test_wav.zipz9https://github.com/joonson/voxconverse/archive/master.zipF
corpus_dirforce_downloadc                 C   s  t | } | jddd | d }| r|rtd tt| d  t| d }|| d  W d    n1 s9w   Y  t	j
| d | d dd t	| d  td	 tt| d
  t| d
 }|| d  W d    n1 suw   Y  t	j
| d | d dd t	| d  td tt| d  t| d }||  W d    n1 sw   Y  t	j
| d | dd t	| d  | d   | d
   | d   |  td d S )NTparentsexist_okz
.completedzDownloading VoxConverse dev setzdev.zipdevz	dev/audio)dirs_exist_okz Downloading VoxConverse test setztest.ziptestztest/voxconverse_test_wavz#Downloading VoxConverse annotationszannotations.zipzvoxconverse-masterDone)r   mkdiris_fileprintr   DEV_AUDIO_ZIPzipfileZipFile
extractallshutilcopytreermtreeTEST_AUDIO_ZIPANNOTATIONS_ZIPunlinktouch)r   r   completed_detectorzip_f r.   N/home/ubuntu/.local/lib/python3.10/site-packages/lhotse/recipes/voxconverse.pydownload_voxconverse!   sL   
r0   
output_dir
split_testc                 C   s  t |  } i }|r7t| d d|d< t| d d}|d t|d  |d< |t|d d  |d< nt| d d|d< t| d d|d< i }| D ]\}}g }g }	|D ]5}
|t|
 |
	d	d}t
t|D ]\}\}}}|	t|
j d| |
j||d	d
|d quq]t|}t|	}t||\}}t|| |d urt |}|jddd ||d| d  ||d| d  ||d||< qS|S )Nr   z*.wavtrainr       z.rttm-r   en)idrecording_idstartdurationchannellanguagespeakerTr   voxconverse_recordings_z	.jsonl.gzvoxconverse_supervisions_)
recordingssupervisions)r   absolutesortedgloblenitemsappendr   	from_filewith_suffix	enumerate
_read_rttmr   stemr   from_recordingsr   from_segmentsr   r   r   to_file)r   r1   r2   splits
test_files	manifestssubsetwavsrA   rB   wav_file	rttm_fileixr:   r;   r>   recording_setsupervision_setr.   r.   r/   prepare_voxconverseS   s\   


r[   c                 c   s    t | d1}| D ]#}| }|dr.| \
}}}}}}}}}}t|t||fV  qW d    d S 1 s:w   Y  d S )NrSPEAKER)open	readlinesstrip
startswithsplitfloat)filenamefline_r:   r;   r>   r.   r.   r/   rL      s   
"rL   )F)NF)*__doc__jsonloggingrer%   r"   collectionsr   concurrent.futuresr   pathlibr   typingr   r   r   r   urllib.errorr	   	tqdm.autor
   lhotser   r   lhotse.audior   r   lhotse.supervisionr   r   r   lhotse.utilsr   r   r   r!   r(   r)   boolr0   r[   rL   r.   r.   r.   r/   <module>   sJ    
4
<