o }oi=z.TextToTextBatch.collate_fn..T batch_first padding_valuecSg|]}|jjdqSrrshaper/r r r!r2>cSr-r rr/r r r!r2?r3cSr7r8rr:r/r r r!r2@r;cSr-r rr/r r r!r2Ar3)r#r$r%r&r')r"rtorchtensorlongr(r*r+r r r! collate_fn:szTextToTextBatch.collate_fnN)r(r)r*rr+rr,r"rrrr staticmethodrCr r r r!r"3s r"c@sTeZdZUded<ded<ded<ded<ded<ded<ded<edddZdS)TextOrAudioToTextBatchr audio_signalsaudio_signal_lengthsr#r$r'r%r&r("List[Union[TextToTextItem, tuple]]r+rr*r,5Union[TextToTextBatch, TextOrAudioToTextBatch, tuple]c Csdd|D}|st||dSdd|D}|s tj|||dStdd|Dddd }td d|D}td d|Dd|d }td d|D}td d|D} tdd|Ddd|Dd|d } tdd|Ddd|D} t||||| | | dS) Collate function for dataloader Can accept mixed batch of text-to-text items and audio-text items (typical for ASR) cSsg|] }t|tr|qSr  isinstancerr/r r r!r2Vz5TextOrAudioToTextBatch.collate_fn..)r(pad_idcSsg|] }t|ts|qSr rLr/r r r!r2[rNrBcSg|]}|dqSr8r r/r r r!r2eTgr4cSrP)r r/r r r!r2frQcSr-r r.r/r r r!r2ir3cSr7r8r9r/r r r!r2kr;cSr-r r>r/r r r!r2lr3cSr-r r<r/r r r!r2or3cSrP)r r/r r r!r2orQcSr7r8r=r/r r r!r2tr;cSrP)r r/r r r!r2trQ)rGrHr#r$r'r%r&)rr"rCrr?r@rArF) r(r+r* text_items asr_itemsrGrHr#r$r'r%r&r r r!rCNs@ z!TextOrAudioToTextBatch.collate_fnN)r(rIr+rr*rr,rJrDr r r r!rFEs rFtextstrr, np.ndarraycCs8t|}tdurtg|}tdur|tt|S)z Helper function for asr tokenization with multiprocessing pool only. Must be defined on the top level. Expects asr_tokenizer_global, asr_bos_id_global, asr_eos_id_global to exist in the current pool process N)asr_tokenizer_global text_to_idsasr_bos_id_globalasr_eos_id_globalappendnpasarray)rWidsr r r!_asr_text_to_tokenss    rbcCstt|S)z Helper function for asr tokenization with multiprocessing pool only. Must be defined on the top level. Expects tts_tokenizer_global to exist in the current pool process )r_r`tts_tokenizer_global)rWr r r!_tts_text_to_tokenssrdfilepathAnyPathIterable[Dict[str, Any]]ccsPt|ddd}|D] }t|}|Vq WddS1s!wYdS)z- Helper function to iterate manifest rzutf-8)encodingN)openjsonloads)reflinerecordr r r!_iterate_manifests "rpcseZdZUdZded<ded<dZded<dZded<d ed < d/d0fd d! Zd1d&d'Zd2d1d)d*Z d+d,Z d-d.Z Z S)3TextToTextDatasetBasezu Base class for loading text-to-text manifests Map-style and Iterable datasets should inherit this class rr*r+Nz Optional[int] asr_bos_id asr_eos_idzList[Dict[str, Any]]datarR@Brmanifest_filepathUnion[AnyPath, List[AnyPath]]speakers_filepath asr_tokenizerrasr_use_start_end_tokenbool tts_parserrtts_text_normalizer 'Normalizer'tts_text_normalizer_call_kwargsr min_words max_wordstokenizer_workers num_partscurrent_part_indexc" szt|rt|dr|j_|rt|dr|j_t|dr%|j_nd_|_ |_ |_ |_ |_ t|trB|d}nt|trJ|g}t}|D]'}tt|d}|tt|Wdn1sqwYqOtt|_tdtjdt|tr|d}nt|tr|g}d d |D_ d}d}g}g}d j D]E}t!t"|D]<}|d }t|}| |kr| ksn||7}|d 7}q|#|d d|vr|#|dq|#|ddqqrt$d|rt$d|d|t|}| d krC|| }|| }||}td| d| d|d||||}|||}|}dd t%|D_&t|dkr[t$ddS| d krt$dt't!fdd|Dt|dD] \}}|j&|d<qvn@dd }t(j)j*||jjf| d!%} t't!| jt+|d"d#t|dD] \}}|j&|d<qWdn 1swY~t,-| d krt$dt't!fd$d|Dt|dD] \}}|j&|d%<qnCrt.d&d'd(}!t(j)j*|!|f| d!%} t't!| jt/|d"d#t|dD] \}}|j&|d%<qWdn 1s1wY~t,-dS))N bos_token eos_token pad_tokenr,rhzLoaded z speakerscSsg|]}t|qSr r)r0rer r r!r2rQz2TextToTextDatasetBase.__init__..FrWrRtts_text_normalizedrTzLTTS normalization is extremely slow! It is recommended to normalize TTS textzSkipped z utterances with zTaking part of the dataset: z index, total z from z to cSsg|]}tqSr )dict)r0_r r r!r2r3zText-to-text dataset is emptyzPreprocessing large text with tokenizer_workers=1 may be slow with TTS tokenizer. Prefer tokenizer_workers=(num_cpu_cores/num_gpus_per_node)c3s|]}|VqdSN)rbr0rWselfr r! sz1TextToTextDatasetBase.__init__..)totalasr_text_tokenscSs"t|at|at|adSr)copydeepcopyrZr\r]) tokenizerbos_ideos_idr r r!_init_asr_tokenize_process$s  zBTextToTextDatasetBase.__init__.._init_asr_tokenize_process) initializerinitargs max_workersi) chunksizec3s|] }j|dVqdS)) normalizeN)rdrneed_normalizationrr r!r@stts_text_tokenszNormalization with tokenizer_workers > 1 is not implemented. It is not recommended to use normalization on the fly at all, since it's extremely slowcSst|adSr)rrrc)rr r r!_init_tts_tokenize_processMszBTextToTextDatasetBase.__init__.._init_tts_tokenize_process)0super__init__hasattrrrrrrsrOr*ryr|tts_normalizertts_normalizer_kwargsr+rMrXsplitrsetrj expanduserupdatemaprreadr_r`sortedr'rinfolenmanifest_pathsrrpr^warningrangert enumerate concurrentfuturesProcessPoolExecutorrbgccollectNotImplementedErrorrd)"rrvrxryrzr|r+r}rrrrrrr'rermnum_skipped_wordsnum_skipped_utterances asr_textsr# manifest_pathtmp_itemrW num_wordsnum_utterancesnum_utterances_partstartenditokenized_textrpoolr __class__rr!rs                         zTextToTextDatasetBase.__init__rWrXr,rYcCsB|j|}|jdur|jg|}|jdur||jt|Sr)ryr[rrrsr^r_r`)rrWrar r r!rb]s      z)TextToTextDatasetBase._asr_text_to_tokensTcCs.|r |jj|fi|j}||}t|Sr)rrrr|r_r`)rrWrtokensr r r!rdes  z)TextToTextDatasetBase._tts_text_to_tokenscCs<|j|}tt|dt|dt|jdS)Nrr)rrr)rtrr? from_numpyrArandomchoicer')rindexr1r r r! __getitem__ks  z!TextToTextDatasetBase.__getitem__cCs t|jSr)rrtrr r r!__len__ss zTextToTextDatasetBase.__len__rRrurRrRrrvrwrxrwryrrzr{r|rr+rr}r~rrrrrrrrrrrrrWrXr,rY)T) rrr__doc__rrrrsrrbrdrr __classcell__r r rr!rqs$     / rqcs4eZdZdZ   ddfdd Zd ddZZS)!TextToTextDatasetz8Text-to-Text Map-style Dataset for hybrid ASR-TTS modelsrRrurvrwrxryrrzr{r|rr+rr}r~rrrrrc s(tj||||||||| | | dd dS)NrR) rvrxryrzr|r+r}rrrrrrr) rrvrxryrzr|r+r}rrrrrr r!rzs zTextToTextDataset.__init__r(rIr,rJcCtj||j|jdSrKrBrFrCr*r+rr(r r r!rC zTextToTextDataset.collate_fn)rRrurR)rvrwrxrwryrrzr{r|rr+rr}r~rrrrrrrrr(rIr,rJ)rrrrrrCrr r rr!rws rcs@eZdZdZ     d#d$fdd ZddZd%d!d"ZZS)&TextToTextIterableDatasetz Text-to-Text Iterable Dataset for hybrid ASR-TTS models Only part necessary for current process should be loaded and stored rRrurrvrwrxryrrzr{r|rr+rr}r~rrrrrrrcs*tj||||||||| | | | | d dS)N) rvrxryrzr|r+r}rrrrrrr)rrvrxryrzr|r+r}rrrrrrrr r!rs z"TextToTextIterableDataset.__init__cCstjj}|durd}t|}nttt|t|j }|j }||}t ||t|}t ||}t j|t|j|S)Nr)r?utilsrtget_worker_inforrmathceilfloat num_workersidminr_arangershufflerr)r worker_inforr per_worker worker_idindicesr r r!__iter__s     z"TextToTextIterableDataset.__iter__r(rIr,rJcCrrrrr r r!rCrz$TextToTextIterableDataset.collate_fnrrr)rrrrrrrCrr r rr!rs rr)rerfr,rg)5 __future__rconcurrent.futuresrrrrkrrpathlibrtypingrrrrr r r r r numpyr_r?torch.utils.datatorch.nn.utils.rnnr tqdm.autor'nemo.collections.asr.data.audio_to_textr"nemo.collections.common.tokenizersrnemo.core.classesrr nemo.utilsr1nemo_text_processing.text_normalization.normalizer ExceptionerXrfrr"rFrbrdrprqrrr r r r!sF  ,        =   U,