o }oi<@sddlZddlZddlZddlmZddlmZmZmZm Z m Z m Z m Z ddl ZddlZddlmZmZddlmZddlmZmZGdddejZGd d d eZGd d d eZGd ddeZGdddeZGdddeZGdddeZ Gddde!Z"GdddeZ#Gddde!Z$Gddde$Z%GdddeZ&Gdd d e&Z'Gd!d"d"eZ(Gd#d$d$e(Z)Gd%d&d&eZ*Gd'd(d(e*Z+Gd)d*d*eZ,Gd+d,d,e,Z-Gd-d.d.eZ.Gd/d0d0e.Z/Gd1d2d2eZ0Gd3d4d4e0Z1Gd5d6d6eZ2Gd7d8d8e2Z3dS)9N) combinations)AnyCallableDictIterableListOptionalUnion)manifestparsers) get_full_path)logging logging_modec@seZdZdZdZdS) _Collectionz%List of parsed and preprocessed data.N)__name__ __module__ __qualname____doc__ OUTPUT_TYPErrk/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/common/parts/preprocessing/collections.pyrsrcs<eZdZdZeddZdeede j ffdd Z Z S)TextzCSimple list of preprocessed text entries, result in list of tokens. TextEntitytokenstextsparsercsRg|j}}|D]}||}|durtd|q|||qt|dS)zInstantiates text manifest and do the preprocessing step. Args: texts: List of raw texts strings. parser: Instance of `CharParser` to convert string to tokens. NzFail to parse '%s' text line.)rr warningappendsuper__init__)selfrrdata output_typetextr __class__rrr(s  z Text.__init__) rrrr collections namedtuplerrstrr CharParserr __classcell__rrr$rr#s $rcsFeZdZdZdedejffdd Zedede efddZ Z S) FromFileTextz6Another form of texts manifest with reading from file.filercs||}t||dS)zInstantiates text manifest and do the preprocessing step. Args: file: File path to read from. parser: Instance of `CharParser` to convert string to tokens. N)_FromFileText__parse_textsrr)r r,rrr$rrr@s zFromFileText.__init__returncCstj|s tdtj|\}}|dkr!t|d}|S|dkr3tddt |D}|St |d }| }Wd|S1sHwY|S)Nz$Provided texts file does not exists!z.csv transcriptz.jsoncss|]}|dVqdS)r#Nr).0itemrrr Usz-FromFileText.__parse_texts..r) ospathexists ValueErrorsplitextpdread_csvtolistlistr item_iteropen readlines)r,_extrfrrr __parse_textsLs   zFromFileText.__parse_texts) rrrrr(r r)r staticmethodrr-r*rrr$rr+=s  r+ceZdZdZejdddZ     ddeedee d ee d ee d ee d ee ed ee edee edee e de j de e de e de ededeffdd ZZS) AudioText@List of audio-transcript text correspondence with preprocessing.AudioTextEntityzGid audio_file duration text_tokens offset text_raw speaker orig_sr langtypename field_namesNFids audio_files durationsroffsetsspeakersorig_sampling_rates token_labelslangsr min_duration max_duration max_numberdo_sort_by_durationindex_by_file_idc! s"|j}d}gdddf\}}}}|ri|_t||||||||| D]\ }}}}}}}}}|dur1d}|durF| durF|| krF||7}|d7}q |dur[| dur[|| kr[||7}|d7}q |durb|}n3|dkrt| dr| jrt|tr|dur}| ||}n td | |}ng}|dur||7}|d7}q ||dur|nd7}||||||||||| |rt j t j |\}} ||jvrg|j|<|j|t |dt || krnq |r|rtd n|jd d d tdt ||dtd||d|s tdt|dS)a Instantiates audio-text manifest with filters and preprocessing. Args: ids: List of examples positions. audio_files: List of audio files. durations: List of float durations. texts: List of raw text transcripts. offsets: List of duration offsets or None. speakers: List of optional speakers ids. orig_sampling_rates: List of original sampling rates of audio files. langs: List of language ids, one for eadh sample, or None. parser: Instance of `CharParser` to convert string to tokens. min_duration: Minimum duration to keep entry with (default: None). max_duration: Maximum duration to keep entry with (default: None). max_number: Maximum number of samples to collect. do_sort_by_duration: True if sort samples list by duration. Not compatible with index_by_file_id. index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. TrNF is_aggregate9lang required in manifest when using aggregate tokenizersLTried to sort dataset by duration, but cannot since index_by_file_id is set.cS|jSNdurationentityrrrz$AudioText.__init__..key1Dataset loaded with %d files totalling %.2f hours+%d files were filtered totalling %.2f hourszRNot all audios have duration information, the total number of hours is inaccurate.rmappingziphasattrr\ isinstancer(r7rr4r5r8basenamelenr rsortinforr)!r rLrMrNrrOrPrQrRrSrrTrUrVrWrXr"all_has_durationr!duration_filtered num_filteredtotal_durationid_ audio_filerboffsetr#speakerorig_srlang text_tokensfile_idr@r$rrresd%       zAudioText.__init__NNNFFrrrrr&r'rrintr(floatrr r)boolrr*rrr$rrF]T       rFcrE) VideoTextz@List of video-transcript text correspondence with preprocessing.rHzGid video_file duration text_tokens offset text_raw speaker orig_sr langrINFrL video_filesrNrrOrPrQrRrSrrTrUrVrWrXc s|j}gdddf\}}}}|ri|_t||||||||| D]\ }}}}}}}}}| dur:|| kr:||7}|d7}q| durK|| krK||7}|d7}q|durR|}n3|dkrvt| drq| jrqt|trq|durm| ||}n td| |}ng}|dur||7}|d7}q||7}||||||||||| |rt j t j |\}}||jvrg|j|<|j|t |dt || krnq|r|rtdn|jd d d td t ||d td||d t|dS)a Instantiates video-text manifest with filters and preprocessing. Args: ids: List of examples positions. video_files: List of video files. durations: List of float durations. texts: List of raw text transcripts. offsets: List of duration offsets or None. speakers: List of optional speakers ids. orig_sampling_rates: List of original sampling rates of audio files. langs: List of language ids, one for eadh sample, or None. parser: Instance of `CharParser` to convert string to tokens. min_duration: Minimum duration to keep entry with (default: None). max_duration: Maximum duration to keep entry with (default: None). max_number: Maximum number of samples to collect. do_sort_by_duration: True if sort samples list by duration. Not compatible with index_by_file_id. index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. rYrNrZr[r\r]r^cSr_r`rarcrrrre4rfz$VideoText.__init__..rgrirjrkrl) r rLrrNrrOrPrQrRrSrrTrUrVrWrXr"r!rvrwrxry video_filerbr{r#r|r}r~rrr@r$rrrsZ%      zVideoText.__init__rrrrr$rrrrcseZdZdZejdddZ        ddeee efde e d e e d e e d e e d e d e de de ffdd ZddZZS)InstructionTuningAudioText5`AudioText` collector from asr structured json files.InstructionTuningTextzjid context context_type context_duration question question_type answer answer_type answer_duration speakerrINFmanifests_filesrTrUmax_seq_lengthrVrWrXdecoder_only_modeluse_phoneme_tokenizerc !sX|j} | |_gdddf\} } } }|ri|_t|D]}|d}|d}|d}|d}|d}|d}|d }|d }|d }|d }|d }|durNdn|}|dkrV|n|}|duri||kri| |7} | d7} q|durz||krz| |7} | d7} qt||||dd}|||d}||||}|r||||ks|||ks||kr| |7} | d7} q||7}| | |||||||||| |rtj tj |\}} d|vr|dd}||jvrg|j|<|j|t | dt | |krnq|r|rt dn| jdddt dt | |dt d| | dt| dS)aHParse lists of audio files, durations and transcripts texts. Args: manifests_files: Either single string file or list of such - manifests to yield items from. *args: Args to pass to `AudioText` constructor. **kwargs: Kwargs to pass to `AudioText` constructor. rYridcontextcontext_duration context_typequestion question_typer|answeranswer_duration answer_typetaskNttsrZg333333?iz.contextir^cSr_r`rarcrrrrerfz5InstructionTuningAudioText.__init__..rgrirjrk)rrrmr r=min_get_lenrr4r5r8rqrrr rrsrtrr)!r rrTrUrrVrWrXrrr"r!rvrwrxr1rrrrrrr|rrrrrbapprox_context_lenapprox_question_lenapprox_answer_lenrr@r$rrrGs     z#InstructionTuningAudioText.__init__cCs\|dkr|dS|dkr|jrt|St|ddS|dkr&t|dStd|d) NSPEECHLTEXT TOKENSzUnknown field type .)rrrsplitr7)r field_typer! duration_datarrrrs z#InstructionTuningAudioText._get_len)NNNNFFFF)rrrrr&r'rr r(rrrrrrrr*rrr$rr<sD   grcs<eZdZdZddeeeefdeeffdd Z Z S) ASRAudioTextrNr parse_funcc sgggggf\}}}}} ggggf\} } } } tj||dD]A}||d||d||d||d| |d| |d| |d| |d | |d qtj||||| | | | | g |Ri|d S) IParse lists of audio files, durations and transcripts texts. Args: manifests_files: Either single string file or list of such - manifests to yield items from. *args: Args to pass to `AudioText` constructor. **kwargs: Kwargs to pass to `AudioText` constructor. rrrzrbr#r{r|r}rRr~Nr r=rrr)r rrargskwargsrLrMrNrrOrPorig_srsrRrSr1r$rrrs<  zASRAudioText.__init__r`) rrrrr r(rrrrr*rrr$rrs0rc@seZdZdZdddZdS)SpeechLLMAudioTextEntityz(Class for SpeechLLM dataloader instance.r.Nc Cs:||_||_||_||_||_||_||_||_| |_dS)zCInitialize the AudioTextEntity for a SpeechLLM dataloader instance.N) rrzrbrrr{r|r}r~) r sidrzrbrrr{r|r}r~rrrrs z!SpeechLLMAudioTextEntity.__init__)r.N)rrrrrrrrrrsrc2eZdZdZdeeeefffdd ZZS) ASRVideoTextz4`VideoText` collector from cv structured json files.rc sgggggf\}}}}}ggggf\} } } } t|D]A} || d|| d|| d|| d|| d| | d| | d| | d| | d qtj|||||| | | | g |Ri|d S) aIParse lists of video files, durations and transcripts texts. Args: manifests_files: Either single string file or list of such - manifests to yield items from. *args: Args to pass to `VideoText` constructor. **kwargs: Kwargs to pass to `VideoText` constructor. rrrbr#r{r|r}rRr~Nr)r rrrrLrrNrrOrPrrRrSr1r$rrrs< zASRVideoText.__init__ rrrrr r(rrr*rrr$rr&rc @seZdZdZ      ddeedeedeedeedeed eed eeed eeed eeed eedeedeede de deefddZ ddZ ddZ dS)SpeechLLMAudioTextzList of audio-transcript text correspondence with preprocessing. All of the audio, duration, context, answer are optional. If answer is not present, text is treated as the answer. NFrLrMrN context_listanswersrOrPrQrSrTrUrVrWrXmax_num_samplesc# sgdddf\}}}|ri|_t||||||||| D]\ }}}}}}}}}|durqt|tr3t|n|}t|tr>t|n|}t|trIt|n|}| dur\|| kr\||7}|d7}q| durm|| krm||7}|d7}q||7}|dur~||7}|d7}qt||||||||| |r|durt j t j |\}} ||jvrg|j|<|j|t dt | krnq|dur|s|t krtdt d|dd|nAtdt d|d|t |t }!fd d tjjt |!d d D}"|"n |dur"|r"td | r6|r.tdnjdddtdt |dtd||d|_dS)aInstantiates audio-context-answer manifest with filters and preprocessing. Args: ids: List of examples positions. audio_files: List of audio files. durations: List of float durations. context_list: List of raw text transcripts. answers: List of raw text transcripts. offsets: List of duration offsets or None. speakers: List of optional speakers ids. orig_sampling_rates: List of original sampling rates of audio files. langs: List of language ids, one for eadh sample, or None. min_duration: Minimum duration to keep entry with (default: None). max_duration: Maximum duration to keep entry with (default: None). max_number: Maximum number of samples to collect. do_sort_by_duration: True if sort samples list by duration. Not compatible with index_by_file_id. index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. rYrNrZzSubsampling dataset from z to z sampleszOversampling dataset from csg|]}|qSrr)r0idxr!rr z/SpeechLLMAudioText.__init__..F)replacezXTried to subsample dataset by max_num_samples, but cannot since index_by_file_id is set.r^cSr_r`rarcrrrrerfz-SpeechLLMAudioText.__init__..rgrirjrk)rmrnrpr<rmaxsumrrr4r5r8rqrrr rtnprandomchoiceextendrrsr!)#r rLrMrNrrrOrPrQrSrTrUrVrWrXrrvrwrxryrzrbr{rrr|r}r~ curr_min_dur curr_max_dur curr_sum_durrr@res_numres_datarrrr)sh%      $    zSpeechLLMAudioText.__init__cCs<|dks |t|jkrtdt|jd|d|j|S)Nrzindex out of range [0,z), got z instead)rrr!r7)r rrrr __getitem__s zSpeechLLMAudioText.__getitem__cCs t|jSr`)rrr!)r rrr__len__s zSpeechLLMAudioText.__len__)NNNFFN) rrrrrrr(rrrrrrrrrrr"sR       f rc steZdZdZ   ddeeeefdeeeeefdedeffd d Zd ed ed e ee ffddZ Z S)SpeechLLMAudioTextCollectionz`SpeechLLMAudioText` collector from SpeechLLM json files. This collector also keeps backward compatibility with SpeechLLMAudioText. Nrrr context_file context_key answer_keyc s||_||_ggggggf\}}} } } } ggg} }}|durht|tr*|dn|}g|_|D]*}t|d}|D]}|}|rK|j |q=Wdn1sVwYq1t d|d|nd|_t j ||jdD]A}| |d| |d| |d | |d | |d | |d | |d | |d| |dqstj||| | | | | ||g |Ri|dS)rN,r3zUse random text context from z for rrrzrbrrr{r|r}r~)rrrpr(rrr>r?striprr rtr r=)_SpeechLLMAudioTextCollection__parse_itemrr)r rrrrrrrLrMrNrrrOrPrrSquestion_file_listfilepathrBliner1r$rrrsh       z%SpeechLLMAudioTextCollection.__init__r manifest_filer.c Cst|}d|vr|d|d<nd|vr|d|d<nd|vr%d|d<|ddur6tj|d|d|d<d|vr>d|d<|j|vrL||j|d<n3d|vrX|d|d<n'd|vr{t|dd }||d<Wdn1suwYnd |d<|j|vr||j|d <nUd |vrt|d d }||d <Wdn1swYn2|j durt j |j  }||d <nd |vrtjd|jd|tjd|d |d <nd|d <t|d|dt|d t|d|dd|dd|dd|ddd}|S)Naudio_filenamerzaudio_filepathrzrrbrr# text_filepathr3narcontext_filepathrz Neither `zC` is found nor`context_file` is set, but found `question` in item: )modezwhat does this audio meanr{r|orig_sample_rater~)rzrbrrr{r|r}r~)jsonloadspopr r rr>readrrrrrrr rrONCEdictr(get)r rrr1rBrrrr __parse_itemsf              z)SpeechLLMAudioTextCollection.__parse_item)Nrr) rrrrr r(rrrrrrr*rrr$rrs&CrcseZdZdZejdddZ     ddeedee d ee e efd ee e d e e d e e d e e de de ffdd ZZS) SpeechLabelz6List of audio-label correspondence with preprocessing.SpeechLabelEntityz audio_file duration label offsetrINFrMrNlabelsrOrTrUrVrWrXc s| ri|_|j} gd} } d} d}t||||D][\}}}}|dur/|dur/||kr/| |7} q|dur@|dur@||kr@| |7} q| | |||||durT| |7} d}| rktjtj|\}}t| d|j|<t| |krsnq|r| r~t dn| j ddd |rt d t| d nt d | d ddt dt| d| d ddt ttdd| |_t dt| t|jt| dS)aInstantiates audio-label manifest with filters and preprocessing. Args: audio_files: List of audio files. durations: List of float durations. labels: List of labels. offsets: List of offsets or None. min_duration: Minimum duration to keep entry with (default: None). max_duration: Maximum duration to keep entry with (default: None). max_number: Maximum number of samples to collect. do_sort_by_duration: True if sort samples list by duration. index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. rYTNFrZr^cSr_r`rarcrrrrejrfz&SpeechLabel.__init__..rgDataset loaded with z( items. The durations were not provided.,Filtered duration for loading collection is rj .2f hours.z!Dataset successfully loaded with z4 items and total duration provided from manifest is cSr_r`)label)xrrrreurfz+# {} files loaded accounting to # {} labels)rmrrnrr4r5r8rqrrr rrsrtsortedsetmap uniq_labelsformatrr)r rMrNrrOrTrUrVrWrXr"r!rvrxduration_undefinedrzrbcommandr{rr@r$rrr-sL    zSpeechLabel.__init__r)rrrrr&r'rrr(rr rrrrr*rrr$rr%s<    rcsXeZdZdZ   d deeeefffdd Zdeded eee ffd d Z Z S) ASRSpeechLabelz3`SpeechLabel` collector from structured json files.FNrcsggggf\}}} } g} tj||jdD]<} || d|| d|s6| d} |s0| n| |}n t| d} | g}| | | | d| |q|rYt| |_ t j ||| | g|Ri|dS)aParse lists of audio files, durations and transcripts texts. Args: manifests_files: Either single string file or list of such - manifests to yield items from. is_regression_task: It's a regression task. cal_labels_occurrence: whether to calculate occurence of labels. delimiter: separator for labels strings. *args: Args to pass to `SpeechLabel` constructor. **kwargs: Kwargs to pass to `SpeechLabel` constructor. rrzrbrr{N) r r=_ASRSpeechLabel__parse_itemrrrrr&Counterlabels_occurrencerr)r ris_regression_taskcal_labels_occurrence delimiterrrrMrNrrO all_labelsr1r label_listr$rrr~s     $zASRSpeechLabel.__init__rrr.cCst|}d|vr|d|d<nd|vr|d|d<ntd|dtj|d|d|d<d|vr [0 1 2 1 0 0 2] In this seq of label , if label do not appear before, assign new relative labels len(pos); else reuse previous assigned relative labels. Args: seq_label (str): A string of a sequence of labels. Return: relative_seq_label (List) : A list of relative sequence of labels unique_labels_in_seq (Set): A set of unique labels in the sequence )rrrrrrkeys)r rseqconversion_dictrelative_seq_labelseg convertedunique_labels_in_seqrrrr s    z,FeatureSequenceLabel.relative_speaker_parserNF)rrrrr&r'rrr(rrrrr r*rrr$rr s$ 0r csbeZdZdZ  ddeeeefdeede ffdd Z d ed ed e ee ffd d Z ZS)ASRFeatureSequenceLabelz@`FeatureSequenceLabel` collector from asr structured json files.NFrrVrXcsRgg}}tj||jdD]}||d||dq t||||dS)aParse lists of feature files and sequences of labels. Args: manifests_files: Either single string file or list of such manifests to yield items from. max_number: Maximum number of samples to collect; pass to `FeatureSequenceLabel` constructor. index_by_file_id: If True, saves a mapping from filename base (ID) to index in data; pass to `FeatureSequenceLabel` constructor. rrrN)r r= _parse_itemrrr)r rrVrXr r r1r$rrr#s z ASRFeatureSequenceLabel.__init__rrr.cCst|}d|vr|d|d<nd|vr|d|d<ntd|dtj|d|d<d|vr;|d|d<ntd|dt|d|dd}|S) Nfeature_filenamerfeature_filepathrz! without proper feature file key.rz without proper seq_label key.)rr)rrrr7r4r5 expanduserrrrrrr<s&   z#ASRFeatureSequenceLabel._parse_itemr)rrrrr r(rrrrrrrrr*rrr$rr s&rcseZdZdZejdddZ   ddeedee d eed ee d ee d ee d ee dee de e dedeffdd ZZS)DiarizationLabelzBList of diarization audio-label correspondence with preprocessing.DiarizationLabelEntityz^audio_file duration rttm_file offset target_spks sess_spk_dict clus_spk_digits rttm_spk_digitsrINFrMrN rttm_filesrOtarget_spks_listsess_spk_dicts clus_spk_list rttm_spk_listrVrWrXc  s| ri|_|j} gd} }t||||||||}|D]=\}}}}}}}}|dur*d}| | ||||||||| rOtjtj|\}}t| d|j|<t| | krWnq| rj| rbt dn| j dddt d |t d t| d t|d t | dS) aFInstantiates audio-label manifest with filters and preprocessing. Args: audio_files: List of audio file paths. durations: List of float durations. rttm_files: List of RTTM files (Groundtruth diarization annotation file). offsets: List of offsets or None. target_spks (tuple): List of tuples containing the two indices of targeted speakers for evaluation. Example: [[(0, 1), (0, 2), (0, 3), (1, 2), (1, 3), (2, 3)], [(0, 1), (1, 2), (0, 2)], ...] sess_spk_dict (Dict): List of Mapping dictionaries between RTTM speakers and speaker labels in the clustering result. clus_spk_digits (tuple): List of Tuple containing all the speaker indices from the clustering result. Example: [(0, 1, 2, 3), (0, 1, 2), ...] rttm_spkr_digits (tuple): List of tuple containing all the speaker indices in the RTTM file. Example: (0, 1, 2), (0, 1), ...] max_number: Maximum number of samples to collect do_sort_by_duration: True if sort samples list by duration index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. rYNrrZr^cSr_r`rarcrrrrerfz+DiarizationLabel.__init__..rg/Filtered duration for loading collection is %f.Total & session files loaded accounting to # audio clips)rmrrnrr4r5r8rqrrr rrsrtrr)r rMrNr!rOr"r#r$r%rVrWrXr"r!rv zipped_itemsrzrb rttm_filer{ target_spks sess_spk_dictclus_spk_digitsrttm_spk_digitsrr@r$rrrbs^)      zDiarizationLabel.__init__NFF)rrrrr&r'rrr(rtuplerrrrrr*rrr$rrZs@    rc sxeZdZdZ   ddeeeefdededeffdd Z dd ed efd dZ dededeee ffddZ Z S)DiarizationSpeechLabelzP`DiarizationLabel` diarization data sample collector from structured json files.Fremb_dictclus_label_dict round_digitsc# sd||_||_||_||_||_ggggggggf\} } } } } }}}tj||jdD]}|jrrtt t dd||dD}|drmt |j }|j||dd}dd| D}d d| D}|jrl|}ned }d }n`g}t|dd #}|D]}|j|d d \}}}|d|||qWd n1swYt }|D] }|d}||qtt |} ddt| D}t| }!|!}|!}t|dkrdg}"n ddt|dD}"|"D]2}!| |d| |d| |d| |d| |!||||||qq)tj| | | | | |||g|Ri|d S)a Parse lists of audio files, durations, RTTM (Diarization annotation) files. Since the diarization model infers only two speakers, speaker pairs are generated from the total number of speakers in the session. Args: manifest_filepath (str): Path to input manifest JSON files. emb_dict (Dict): Dictionary containing cluster-average embeddings and speaker mapping information. clus_label_dict (Dict): Segment-level speaker labels from clustering results. round_digit (int): Number of digits to round. seq_eval_mode (bool): If True, F1 score will be calculated for each speaker pair during inference mode. pairwise_infer (bool): If True, this dataset class operates in inference mode. In inference mode, a set of speakers in the input audio is split into multiple pairs of speakers and speaker tuples (e.g., 3 speakers: [(0,1), (1,2), (0,2)]) and then fed into the diarization system to merge the individual results. *args: Args to pass to `SpeechLabel` constructor. **kwargs: Kwargs to pass to `SpeechLabel` constructor. rcSsg|]}|dqSr3rr0rrrrrrz3DiarizationSpeechLabel.__init__..uniq_idr+rmcSs$i|]\}}t|dd|qS)r@rrr0kvrrr s$z3DiarizationSpeechLabel.__init__..cSs"g|] \}}t|ddqS)r@rZr;r<rrrrs"Nr3r)decimalsz{} {} {}r:cSsi|]\}}||qSrr)r0rhvalrrrr?sr3)rrZcSsg|]}|qSrrr8rrrrsrzrbr{)r6r4r5 seq_eval_modepairwise_inferr r=(_DiarizationSpeechLabel__parse_item_rttmrr<rrritemsr>r?split_rttm_linerrradd enumerater1rrrrr)#r rr4r5r6rBrCrrrMrNr!rOr"r#r$r%r1clus_speaker_digitsbase_scale_index_sess_spk_dictr-rttm_speaker_digits rttm_labelsrBrstartendr| speaker_set rttm_linespk_str speaker_listr, spk_comb_listr$rrrs# "             zDiarizationSpeechLabel.__init__rrQr@cCsT|}tt|d|}tt|d|tt|d|}|d}|||fS)a Convert a line in RTTM file to speaker label, start and end timestamps. An example line of `rttm_line`: SPEAKER abc_dev_0123 1 146.903 1.860 speaker543 The above example RTTM line contains the following information: session name: abc_dev_0123 segment start time: 146.903 segment duration: 1.860 speaker label: speaker543 Args: rttm_line (str): A line in RTTM formatted file containing offset and duration of each segment. decimals (int): Number of digits to be rounded. Returns: start (float): Start timestamp in floating point number. end (float): End timestamp in floating point number. speaker (str): speaker string in RTTM lines. r)rrroundr)r rQr@rttmrNrOr|rrrrF7s $ z&DiarizationSpeechLabel.split_rttm_linerrr.c Cst|}d|vr|d|d<nd|vr|d|d<ntd|dtj|d|d<tjtj|dd|d<d|vrKtd|d t |d|d|d|d | d d d }|S)2Parse each rttm file and save it to in Dict formatrrzrrrrr9rbr rttm_filepathr{Nrzr9rbr+r{) rrrr7r4r5rr8rqrrrrrr__parse_item_rttmXs(    z(DiarizationSpeechLabel.__parse_item_rttm)r3FF)r)rrrrr r(rrrrrFrrDr*rrr$rr2s o&!r2cspeZdZdZejdddZ   ddeedeed ee d eed ee d e e d e de ffdd Z ZS)EndtoEndDiarizationLabelzMList of end-to-end diarization audio-label correspondence with preprocessing.r z,audio_file uniq_id duration rttm_file offsetrINFrMuniq_idsrNr!rOrVrWrXc  s"|ri|_|j} gd} } t|||||} | D]I\} }}}}|dur$d}| | | |||||rXt| trCt| dkrCtd| tj tj | \}}t| d|j|<t| |kr`nq|rs|rkt dn| jddd t d | t d t| d t|d t| dS)a Instantiates audio-label manifest with filters and preprocessing. This method initializes the EndtoEndDiarizationLabel object by processing the input data and applying optional filters and sorting. Args: audio_files (List[str]): List of audio file paths. uniq_ids (List[str]): List of unique identifiers for each audio file. durations (List[float]): List of float durations for each audio file. rttm_files (List[str]): List of RTTM path strings (Groundtruth diarization annotation file). offsets (List[float]): List of offsets or None for each audio file. max_number (Optional[int]): Maximum number of samples to collect. Defaults to None. do_sort_by_duration (bool): If True, sort samples list by duration. Defaults to False. index_by_file_id (bool): If True, saves a mapping from filename base (ID) to index in data. Defaults to False. rYNrzEmpty audio file list: rZr^cSr_r`rarcrrrrerfz3EndtoEndDiarizationLabel.__init__..rgr&r'r(r))rmrrnrrpr<rrr7r4r5r8rqr rrsrtrr)r rMr^rNr!rOrVrWrXr"r!rvr*rzr9rbr+r{rr@r$rrrysT      z!EndtoEndDiarizationLabel.__init__r0rrrrr&r'rrr(rrrrrr*rrr$rr]qs4  r]csXeZdZdZ d deeeefdeffdd Zdeded e ee ffd d Z Z S) EndtoEndDiarizationSpeechLabelzPEnd-to-end speaker diarization data sample collector from structured json files.r3rr6c s||_gggggf\}}}}} tj||jdD]%} || d|| d|| d|| d| | dqtj||||| g|Ri|dS)a$ Parse lists of audio files, durations, RTTM (Diarization annotation) files. Since diarization model infers only two speakers, speaker pairs are generated from the total number of speakers in the session. Args: manifest_filepath (str): Path to input manifest json files. round_digit (int): Number of digits to be rounded. *args: Args to pass to `SpeechLabel` constructor. **kwargs: Kwargs to pass to `SpeechLabel` constructor. rrzr9rbr+r{N)r6r r=0_EndtoEndDiarizationSpeechLabel__parse_item_rttmrrr) r rr6rrrMr^rNr!rOr1r$rrrs0 z'EndtoEndDiarizationSpeechLabel.__init__rrr.c Cst|}d|vs|ddurd|d<d|vr d|vr$|d|d<nd|vr0|d|d<ntd|dt|dtrT|dD] }tt||d qCt|d<n0t|dt rwt|d|d |d<t j |dsvt d |dn td|d |dd d |vrnd|vr|d|d <nd|vr|d|d <nd|d <|d durt|d |d |d <t j |d st d|d d|vrt j t j |dd|d<t|dt std|dd|vrtd|dt|d|d|d|d |ddd}|S)rYr{NrrzrrrrrzAudio file not found: z" without proper audio file value: rr+ rttm_filenamerZzRTTM file not found: r9z without proper uniq_id key.rbrr[)rrrr7rpr<audio_file_listrr r(r4r5r6FileNotFoundErrorr8rqrr)r rrr1single_audio_filerrrr\sj       z0EndtoEndDiarizationSpeechLabel.__parse_item_rttmr7) rrrrr r(rrrrrrar*rrr$rr`s&/r`cs~eZdZdZejdddZ    ddeee e fdee dee d ee d e e d e e d e e d e ffdd ZZS)Audioz8Prepare a list of all audio items, filtered by duration.z audio_files duration offset textrINFaudio_files_list duration_list offset_list text_listrTrUrVrWc s|j} gd} } d\} } t||||D]>\}}}}|dur*||kr*| |7} | d7} q|dur;||kr;| |7} | d7} q| |7} | | ||||t| |krQnq|r\| jdddtdt| | d td | | d t| dS) aLInstantiantes an list of audio files. Args: audio_files_list: list of dictionaries with mapping from audio_key to audio_filepath duration_list: list of durations of input files offset_list: list of offsets text_list: list of texts min_duration: Minimum duration to keep entry with (default: None). max_duration: Maximum duration to keep entry with (default: None). max_number: Maximum number of samples to collect. do_sort_by_duration: True if sort samples list by duration. rY)rrYNrZcSr_r`rarcrrrrexrfz Audio.__init__..rgrirjrk) rrnrrrrsr rtrr)r rgrhrirjrTrUrVrWr"r!rxrwrvrMrbr{r#r$rrrHs,  zAudio.__init__)NNNF)rrrrr&r'rrrr(rrrrrr*rrr$rrfCs0 rfcs\eZdZdZdeeeefdeeefffdd Zdededeee ffd d Z Z S) AudioCollectionz)List of audio files from a manifest file.manifest_filesaudio_to_manifest_keyc st|tkr |d}|D]\}}t|tkr$d|vr$|d||<q||_ggggf\}}} } tj||jdD]} || d|| d| | d| | dq:t j ||| | g|Ri|dS)aInstantiates a list of audio files loaded from a manifest file. Args: manifest_files: path to a single manifest file or a list of paths audio_to_manifest_key: dictionary mapping audio signals to keys of the manifest rrrMrbr{r#N) typer(rrErmr r=_AudioCollection__parse_itemrrr) r rlrmrr audio_key manifest_keyrgrhrirjr1r$rrrs  $zAudioCollection.__init__rrr.c sdtdttttffdd}t|}i}|jD]A\}}|||}t|tr2t |||<qt|t rCfdd|D||<q|durQ| drQd||<qt d t|d |||d <d |vrot d |dd|vrwd|d<d|vrd|d<t|d |d |d|ddS)a;Parse a single line from a manifest file. Args: line: a string representing a line from a manifest file in JSON format manifest_file: path to the manifest file. Used to resolve relative paths. Returns: Dictionary with audio_files, duration, and offset. r1rqcSs|durd}|St|tr||}|St|trHg}|D])}||}t|tr-||qt|tr7||7}qtdt|d|d||Stdt|d|)z{Get item[key] if key is string, or a list of strings by combining item[key[0]], item[key[1]], etc. NUnexpected type z of item for key z: z of manifest_key: )rpr(rrr<r7rn)r1rqrzrhitem_keyrrrget_audio_files"      z4AudioCollection.__parse_item..get_audio_filecsg|]}t|qSr)r r )r0rBrrrrsz0AudioCollection.__parse_item..Nrrrz of audio_file: rMrbz Duration not available in line: z. Manifest file: r{rYr#)rMrbr{r#)rr r(rrrrmrErpr r r startswithr7rnr) r rrrtr1rMrprqrzrrurrs,     zAudioCollection.__parse_item) rrrrr r(rrrrror*rrr$rrks &%rkcsteZdZdZejdddZ     ddeedeed ee d e e d e e d e e d e de ffdd Z ZS) FeatureLabelzKList of feature sequence and their label correspondence with preprocessing.FeatureLabelEntityzfeature_file label durationrINFr rrNrTrUrVrWrXc sd|j} g} d} d} t|_|ri|_t|||D]T\} }}|dur*||kr*| |7} q|dur7||kr7| |7} q| | | |||jt|O_| |7} |rdtjtj | \}}t | d|j|<t | |krlnq|r|rwt dn| j dddt d| d d d t d t | d | ddd t dt | t |jt| dS)aXInstantiates feature-SequenceLabel manifest with filters and preprocessing. Args: feature_files: List of feature files. labels: List of labels. max_number: Maximum number of samples to collect. index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. rYNrZr^cSr_r`rarcrrrre7rfz'FeatureLabel.__init__..rgri( z.2frrz items, total duration of rjrz.# {} files loaded including # {} unique labels)rrrrmrnrr4r5r8rqrrr rrsrtrrr)r r rrNrTrUrVrWrXr"r!rvrxrrrbrr@r$rrrs>  "zFeatureLabel.__init__rr_rrr$rrws8  rwc sheZdZdZ   ddeeeefdededeeffdd Z d ed ed e ee ffd dZ Z S)ASRFeatureLabelz8`FeatureLabel` collector from asr structured json files.FNrrrrcsggg}}} g} tj||jdD]5} || d| | d|s4| d} |s.| n| |} n t| d} | g} || | | q|rPt| |_ t j ||| g|Ri|dS)aParse lists of feature files and sequences of labels. Args: manifests_files: Either single string file or list of such - manifests to yield items from. max_number: Maximum number of samples to collect; pass to `FeatureSequenceLabel` constructor. index_by_file_id: If True, saves a mapping from filename base (ID) to index in data; pass to `FeatureSequenceLabel` constructor. rrrbrN) r r=rrrrrr&rrrr)r rrrrrrr rrNrr1rrr$rrrBs    "zASRFeatureLabel.__init__rrr.cCst|}d|vr|d|d<nd|vr|d|d<n d|vr)td|dtj|d|d|d<d|vr@|d|d<ntd|dt|d|d|d d }|S) Nrrrrz# without proper 'feature_file' key.rrz without proper 'label' key.rb)rrrb)rrrr7r r rrrrrris  zASRFeatureLabel._parse_itemr)rrrrr r(rrrrrrrr*rrr$rry?s&'ryc!seZdZdZejdddZ     ddeedee d ee d ee d ee d ee d ee edee edee edee e de j de e de e de ededef fdd ZZS) FeatureTextrGFeatureTextEntityzSid feature_file rttm_file duration text_tokens offset text_raw speaker orig_sr langrINFrLr r!rNrrOrPrQrRrSrrTrUrVrWrXc"s|j}gdddf\}}}}|ri|_t||||||||| | D]\ }}}}}}}}} }| dur<|| kr<||7}|d7}q| durM|| krM||7}|d7}q| durT| }n3|dkrxt| drs| jrst|trs|duro| ||}n td| |}ng}|dur||7}|d7}q||7}|||||||||||| |rt j t j |\} }!| |jvrg|j| <|j| t |dt ||krnq|r|rtdn|jd d d td t ||d td||d t|dS)aKInstantiates feature-text manifest with filters and preprocessing. Args: ids: List of examples positions. feature_files: List of audio feature files. rttm_files: List of audio rttm files. durations: List of float durations. texts: List of raw text transcripts. offsets: List of duration offsets or None. speakers: List of optional speakers ids. orig_sampling_rates: List of original sampling rates of audio files. langs: List of language ids, one for eadh sample, or None. parser: Instance of `CharParser` to convert string to tokens. min_duration: Minimum duration to keep entry with (default: None). max_duration: Maximum duration to keep entry with (default: None). max_number: Maximum number of samples to collect. do_sort_by_duration: True if sort samples list by duration. Not compatible with index_by_file_id. index_by_file_id: If True, saves a mapping from filename base (ID) to index in data. rYrNrZr[r\r]r^cSr_r`rarcrrrrerfz&FeatureText.__init__..rgrirjrkrl)"r rLr r!rNrrOrPrQrRrSrrTrUrVrWrXr"r!rvrwrxry feat_filer+rbr{r#r|r}r~rrr@r$rrrsp'       zFeatureText.__init__rrrrr$rrzsX      rzcr)ASRFeatureTextz7`FeatureText` collector from asr structured json files.rc sggggggf\}}}}}} ggggf\} } } } t|D]H}||d||d||d||d||d| |d| |d| |d| |d | |d qtj|||||| | | | | g |Ri|d S) rrrr+rbr#r{r|r}rRr~Nr)r rrrrLr r!rNrrOrPrrRrSr1r$rrrsT   zASRFeatureText.__init__rrrr$rr}rr})4r&rr4 itertoolsrtypingrrrrrrr numpyrpandasr9+nemo.collections.common.parts.preprocessingr r 4nemo.collections.common.parts.preprocessing.manifestr nemo.utilsr rUserListrrr+rFrrrobjectrrrrrrr rrr2r]r`rfrkrwryrzr}rrrrsL $  sl+*vVOV:k-Zx=xGCy