o Si:#@sdZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZddlmZddlmZmZmZddlmZmZdd lmZdd lmZmZdd lmZmZdd l m!Z!m"Z"d Z#dddDZ$dZ%dZ&e'(e%e&Z)ddZ*ddZ+   d'de!de ee'e e'fde,defddZ- d(d e!d!e e!de,de e'e e'eeeffffd"d#Z.d$e de e eeffd%d&Z/dS))a ReazonSpeech is an open-source dataset that contains a diverse set of natural Japanese speech, collected from terrestrial television streams. It contains more than 35,000 hours of audio. The dataset is available on Hugging Face. For more details, please visit: Dataset: https://huggingface.co/datasets/reazon-research/reazonspeech Paper: https://research.reazon.jp/_static/reazonspeech_nlp2023.pdf N) defaultdict)Path)AnyDictOptionalSequenceTupleUnion)tqdm)CutSet fix_manifests$validate_recordings_and_supervisions) Recording RecordingSet) parallel_map)manifests_existread_manifests_if_cached)SupervisionSegmentSupervisionSet)Pathlikeis_module_available)tinysmallmediumlargeallsmall-v1z medium-v1zall-v1cCsi|]}t|dqS))ord).0xr!O/home/ubuntu/.local/lib/python3.10/site-packages/lhotse/recipes/reazonspeech.py &sr#u!、。「」『』，,？！!!?!?uａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺ０１２３４５６７８９>abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789csDtdr ddlntd|tt}fdd}td||S)z Convert full-width characters to half-width, and remove punctuations. :param s: str, input string. :return: str, normalized string. num2wordsrNz]To process the ReazonSpeech corpus, please install optional dependency: pip install num2wordscsj|dddS)Nrja)lang)r%group)mr%r!r"9sznormalize..z \d+\.?\d*)rr% ImportError translate PUNCTUATIONSZEN2HANresub)sconvr!r*r" normalize,s  r4cCsFt|ddd}tj||dddWddS1swYdS)z Writes data to a JSON file. :param data: The data to write. :param filename: The name of the file to write to. wutf-8encodingF) ensure_asciiindentN)openjsondump)datafilenamefr!r!r" write_to_json=s"rB.auto target_dir dataset_partsnum_jobsreturnc stdrddlddlm}m}ntdt|}|jddd|d}|d kr+d }nt|t r3|g}|D]}t d ||d |d||d d}q5dt dt dt ffdd }|d|dd}|j|d|j|d}|j|d|ddd|jd|S)aY Download the ReazonSpeech dataset. :param target_dir: Pathlike, the path of the dir to storage the dataset. :param dataset_parts: the parts of the dataset to download (e.g. small, medium, or large). :param num_jobs: the number of processes to download and format. :return: the path to downloaded data and the JSON file. datasetsrN)Audio load_datasetzhTo process the ReazonSpeech corpus, please install optional dependencies: pip install datasets soundfileTparentsexist_ok ReazonSpeechrD)rzDownloading ReazonSpeech part: zreazon-research/reazonspeech)trust_remote_code cache_dirnum_proctrainexampleidxrIcsHt||d<|dd|d<t|d|d<|ddj|d<|S)Nidaudiopathaudio_filepath transcriptiontextduration)strr4infor])rUrVsfr!r"format_exampleos z-download_reazonspeech..format_examplerX)decode) with_indicesremove_columnsrS dataset.jsonFr9)rS force_asciir;lines batch_size)r soundfilerJrKrLr,rmkdir isinstancer^loggingr_dictint cast_columnmap column_namesto_jsonnum_rows) rFrGrHrKrL corpus_dirpartdsrbr!r`r"download_reazonspeechHsR    rxru output_dirc Cst|}|sJd|t|dddd4}t|}|dd}|dd}|dd}t||d t||d t||d Wdn1sOwYd }t|}|jd d dt||ddd d} |D]} t d| t | |dddrt d| dql|| d} t| ddd }t|} Wdn1swYt |d| d} t |d| df}t |d| dJ}ttt| |dddD]6\}}tt |gt |gd\}}t||dtj||d}| |d||d||dqWdn 1swYWdn 1s,wYWdn 1ssX          F "^