o
    i                     @   s   d dl Z d dlZeje   d dlZd dlmZ d dlm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ dd Zd	d
 Zedkr}dZdZdZdZg ded  d e Zg de d Zee	ddde  Zede de d e  dS dS )    N)ProcessPoolExecutor)files)Dataset)tqdm)convert_char_to_pinyinc              	   C   s(  t d|  tj| |d}tj| |d}t|}g g g }}}t|D ]f}ttj||ddd}	|	 d}
W d    n1 sHw   Y  |
d }tj||d	 }|
d
 	 }|
| tdkrt|t|gtd n	tdkr}|
| t|\}}|
|jd |  q(|||fS )NzDealing with: txtswavsrutf-8encoding	r   z.wav   pinyin)	polyphonechar)printospathjoinlistdirr   openreadlinesplitstripappend	tokenizerextendr   r   
torchaudioloadshape)dataset_pathsub_pathtext_dir	audio_dir
text_filesaudio_pathstexts	durations	text_filefile
first_lineaudio_nm
audio_pathtextaudiosample_rate r2   a/home/ubuntu/.local/lib/python3.10/site-packages/f5_tts/train/datasets/prepare_wenetspeech4tts.pydeal_with_sub_path_files   s(   



r4   c               	      s.  t dv sJ g g g } }}ttd}g }tD ] t } fdd|D }|D ]}||t | q)qt	|t
|dD ]}| \}	}
}| |	 ||
 || q?|  tjdsgtd tdt d t| ||d	}|jt d
dd tt dddd}tjd|i|dd W d    n1 sw   Y  td t }t	|D ]	}|t| qt dkr|dd tddD dd tddD   tt dd}t|D ]	}||d  qW d    n1 sw   Y  tdt dt
|  tdt d t
| d d S )!N)r   r   )max_workersc                    s&   g | ]}t jt j |r|qS r2   )r   r   isdirr   ).0itemr"   r2   r3   
<listcomp>:   s   & zmain.<locals>.<listcomp>)totaldataz
Saving to z ...)r.   r/   durationz/raw2GB)max_shard_sizez/duration.jsonwr
   r   r=   F)ensure_asciizF
Evaluating vocab size (all characters and symbols / all phonemes) ...r   c                 S   s   g | ]}t |qS r2   )chr)r7   ir2   r2   r3   r:   W   s                 z
/vocab.txt
z
For z, sample count: zFor z, vocab size is: ) r   r   r5   dataset_pathsr   r   r   submitr4   r   lenresultr   shutdownr   existsmakedirsr   save_dirr   	from_dictsave_to_diskr   jsondumpsetupdatelistrangesortedwritedataset_name)audio_path_list	text_listduration_listexecutorfutures	sub_items	sub_pathsr#   futurer'   r(   r)   datasetftext_vocab_setr/   vocabr2   r9   r3   main1   sN   





.rh   __main__rD   r   Tr   )WenetSpeech4TTS_PremiumWenetSpeech4TTS_StandardWenetSpeech4TTS_Basic_)z!<SOME_PATH>/WenetSpeech4TTS/Basicz$<SOME_PATH>/WenetSpeech4TTS/Standardz#<SOME_PATH>/WenetSpeech4TTS/Premiumf5_ttsz../../z/data/z
Choose Dataset: z, will save to rH   )r   sysr   r   getcwdrS   concurrent.futuresr   importlib.resourcesr   r   datasetsr   r   f5_tts.model.utilsr   r4   rh   __name__r5   r   r   dataset_choicer[   rI   strjoinpathrP   r   r2   r2   r2   r3   <module>   s<   /
