o }oi$]@sJddlZddlZddlZddlZddlZddlmZddlm Z m Z m Z m Z ddl Z ddlmZddlmZmZddlmZddlmZddlmZdd lmZdd lmZdd lmZmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+ddl,m-Z-ddl.m/Z/m0Z0dgZ1dZ2dZ3dZ4ddZ5Gddde j6j7e+eZ8dS)N)deepcopy)AnyListOptionalUnion)rank_zero_only) DictConfig OmegaConf)tqdm) score_labels)EncDecClassificationModel)EncDecSpeakerLabelModel)DiarizationMixin)audio_rttm_mapget_embs_and_timestampsget_uniqname_from_filepathparse_scale_configsperform_clustering)segments_manifest_to_subsegments_manifestvalidate_vad_manifestwrite_rttm2manifest)generate_overlap_vad_seqgenerate_vad_segment_tableget_vad_stream_statusprepare_manifest)Model)SaveRestoreConnector)logging model_utilsClusteringDiarizerzmodel_config.yamlzvad_model.nemozspeaker_model.nemocCs|}ttdd|S)z/lists available pretrained model names from NGCcSs|jSN)pretrained_model_name)xr#c/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/asr/models/clustering_diarizer.py>sz+get_available_model_names..)list_available_modelslistmap) class_nameavailable_modelsr#r#r$get_available_model_names;sr+c seZdZdZd/deeefffdd ZeddZ dd Z d/d d Z d d Z ddZ ddZd0dededefddZddZdededefddZd1d eed!efd"d#Zed$efd%d&Ze  d2d'ed(eed)eejfd*d+Zed,efd-d.ZZ S)3ra Inference model Class for offline speaker diarization. This class handles required functionality for diarization : Speech Activity Detection, Segmentation, Extract Embeddings, Clustering, Resegmentation and Scoring. All the parameters are passed through config file Ncfgcstt|trt|}t|}||_|jj|_ d|_ |j j s6|jjj j dur6|jjj j|_|i|_|||jjjj|_|j jj|_dS)NF)super__init__ isinstancerr#convert_model_config_to_dict_configmaybe_update_config_version_cfgdiarizer_diarizer_params has_vad_model oracle_vadvad model_path parameters _vad_params_init_vad_model$multiscale_embeddings_and_timestamps_init_speaker_modelspeaker_embeddings_speaker_params clustering_cluster_params)selfr, speaker_model __class__r#r$r.Is      zClusteringDiarizer.__init__cCsdSr r#)clsr#r#r$r&csz(ClusteringDiarizer.list_available_modelscCs|jjjj}|drtj||jjd|_t d |n"|t tvr.t d |d}t d |tj||jjd|_|jj|_|jj|_d|_d S) zT Initialize VAD model with model name or path passed through config .nemo map_locationz VAD model loaded locally from {}Crequested {} model name not available in pretrained models, insteadvad_telephony_marblenet$Loading pretrained {} model from NGC model_namerITN)r2r3r7r8endswithr restore_fromdevice _vad_modelrinfoformatr+warningfrom_pretrainedr:window_length_in_sec_vad_window_length_in_secshift_length_in_sec_vad_shift_length_in_secr5)rBr8r#r#r$r;gs      z"ClusteringDiarizer._init_vad_modelcCs|dur|jjdurtjr|td|_n`||_n\|jjjj }|dur?| dr?t j ||jjd|_t d|n:| drWt j||jjd|_t d|n"|tt vrgt d|d}t d |t j||jjd |_t|jjjj|jjjj|jjjj|_dS) zb Initialize speaker embedding model with model name or path passed through config NcudarGrHz&Speaker Model restored locally from {}z.ckptrJ ecapa_tdnnrLrM)r2rQtorchr[ is_availableto_speaker_modelr3r>r8rOr rPrrSrTload_from_checkpointr+rUrVrr4r9rWrYmultiscale_weightsmultiscale_args_dict)rBrCr8r#r#r$r=}s6       z&ClusteringDiarizer._init_speaker_modelc Cs@||jj|jdddg|j|jd|jjd }|jj|ddS)N batch_sizeTinferF) manifest_filepath sample_raterd vad_streamlabelsrWrY trim_silence num_workers)test_data_config)r2rggetrXrZrkrRsetup_test_data)rBmanifest_vad_input vad_dl_configr#r#r$_setup_vad_test_datas  z'ClusteringDiarizer._setup_vad_test_datacCs2||jj|jddd|jjd}|j|dS)NrdF)rfrgrdrjrirk)r2rgrmrkr`rn)rB manifest_file spk_dl_configr#r#r$_setup_spkr_test_datas z(ClusteringDiarizer._setup_spkr_test_datac s<tjjddtjjtjj }t|d}||}d}g}t |dddD]}t |d}| t|q0t|} ttjd dj d D]\} } fd d | D} tjjjjj| d| d d} tj| dd} | ddd f}| | dkr|d| }n| | dkr||| }n| | dkr||d}n|}|t|7}tjj|| d}t |ddd}tt|D] }|d||qWdn1swYWdn1swY~ | | dks | | dkr d}qTj j!sj_"j }nt#$dt%jj j!j j&jj j'j(d}|_"d}t#$dt)j t*t+frHj nj +}t,j"||j'j(jd}i}j-D]2}tj.tj||drt/j-|||<tj||d||d<q_t#0d |d!q_t1|j2j2_3dS)"aw Run voice activity detection. Get log probability of voice activity detection and smoothes using the post processing parameters. Using generated frame level predictions generated manifest file for later speaker embedding extraction. input: manifest_file (str) : Manifest file containing path to audio file and label as infer T ignore_errorsrrutf-8encodingaudio_filepathr7descleavedisablecg|] }|jjqSr#)r_rRrQ.0r"rBr#r$ z/ClusteringDiarizer._run_vad.. input_signalinput_signal_lengthdimNstartnextendz.frameaz {0:0.4f} singlez6Generating predictions with overlapping input segments)frame_pred_dirsmoothing_methodoverlaprWrYrkg{Gz?z\Converting frame level prediction to speech/no-speech segment in start and end times format.) vad_pred_dirpostprocessing_paramsframe_length_in_secrkout_dirz.txt rttm_filepathzno vad file found for z! due to zero or negative duration)4shutilrmtree_vad_dirosmakedirsrRevalintrXrZopenjsonloadsappendrr enumerater test_dataloaderverboser]ampautocastrQtypesoftmaxlenpathjoinrangewriterTr: smoothingrrrSrrr2rkr/rdictrAUDIO_RTTM_MAPexistsrrUr _vad_out_file_speaker_manifest_path)rBrr time_unittrunctrunc_lall_lendatalinefilestatusi test_batch log_probsprobspredto_saveoutpathfoutfrsmoothing_pred_dir vad_params table_out_dirAUDIO_VAD_RTTM_MAPkeyr#rr$_run_vads           "   zClusteringDiarizer._run_vadwindowshift scale_tagcCsTtj|jd|d|_td|ddd|jt|j |j||d|_dS)N subsegmentsz.jsonz)Subsegmentation for embedding extraction:_ z, )segments_manifest_filesubsegments_manifest_filerr) rrr _speaker_dirsubsegments_manifest_pathrrSreplacerr)rBrrrr#r#r$_run_segmentationsz$ClusteringDiarizer._run_segmentationcCs|jr=d|_d|_|jj}|jr-tdtd||j|j|j j |jj d}t |}nt d||||n(|jjjdurK|jjj|_n|jjratj|jd|_t|j|j|_ntd t|j|jd dS) z Checks for type of speech activity detection from config. Choices are NeMo VAD, external vad manifest and oracle VAD (generates speech activity labels from provided RTTM files) T2z0Split long audio file to avoid CUDA memory issuez>Try smaller split_duration if you still have CUDA memory issue)inputrWsplit_durationrkrz_If you encounter CUDA memory issue, try splitting manifest entry by split_duration to avoid it.Nzoracle_vad_manifest.jsonzgOnly one of diarizer.oracle_vad, vad.model_path or vad.external_vad_manifest must be passed from config) vad_manifest)r5 _auto_split_split_durationr4rfrrSdebugrXr2rkrrrUrqrr7external_vad_manifestrr6rrrrrr ValueErrorr)rBroconfigr#r#r$"_perform_speech_activity_detection$s8     z5ClusteringDiarizer._perform_speech_activity_detectionrr scale_idx num_scalesc shtd|i_ji_tdg}t j d|dd|ddj dD]M}fd d |D}|\}}}} tj jjj*jj||d \} } | jd } | d | } tj|| fdd }Wdn1sxwY~q1t|dddd} t| D]U\}}|}t|}t|d}|jvrtj|||dd fj|<n ||dd j|<|jvrgj|<|d}||d}j|||gqWdn1swYjjr2t j!"j#d}t j!$|s t j%|ddt|}t j!"||}|d_&t'(jtj&dtd)|dSdS)z This method extracts speaker embeddings from segments passed through manifest_file Optionally you may save the intermediate speaker embeddings for debugging or any use. z%Extracting embeddings for Diarizationr[r/z] extract embeddingsTr}crr#)r_r`rQrrr#r$r^rz:ClusteringDiarizer._extract_embeddings..rrrNrxryrzr|offsetduration embeddingsexist_okz_embeddings.pklwbzSaved embedding files to {})*rrSrtrr`r time_stampsr]emptyr rrrrrQrforwardshapeviewcatcpudetachrr readlinesstriprrrrr?save_embeddingsrrrrrr_embeddings_filepkldumprT)rBrrrrall_embsr audio_signalaudio_signal_lenrislicesrembs emb_shapemanifestrrdic uniq_namerr embedding_dirprefixnamer#rr$_extract_embeddingsLs\           (     z&ClusteringDiarizer._extract_embeddingsrpaths2audio_filesrdc Cs|jj|_tj|jjd|_tj|jr#t dt j |jddt |jtj|js6t |jtj|jd|_tj|jd|_|rN||j_|rmt|turitj|jd|j_|||jjntdt|jj|_tj|jd }tj |dd ||jd }|D]$\}\}}|j||d |d ||j|t||j |j!g|j"|<qt#|j"|j}t$||j||j%|j&j'|j(d\} } t)d*tj+|jjt,|j| | |jj-|jj.|j(dS)a Diarize files provided through paths2audio_files or manifest file input: paths2audio_files (List[str]): list of paths to file containing audio file batch_size (int): batch_size considered for extraction of speaker embeddings and VAD computation speaker_outputsz.Deleting previous clustering diarizer outputs.Tru vad_outputsz vad_out.jsonzpaths2audio_filepath.jsonzMpaths2audio_files must be of type list of paths to file containing audio file pred_rttmsr scale_dict_scale)r)embs_and_timestampsr out_rttm_dirclustering_paramsrQrz!Outputs are saved in {} directory)collarignore_overlapr)/r4r_out_dirrrrrrrrUrrrmkdirrrr2rdrr'rfpath2audio_files_to_manifestrrrrrcitemsrr rrrrr<rrrAr`rQrrSrTabspathr rr) rBr rdrscalesrrrr all_referenceall_hypothesisr#r#r$diarizes\      zClusteringDiarizer.diarize save_pathcCst:}tj|t}tj|t}|j|d|jr)tj|t }|j ||j |t j||dWddS1sAwYdS)a Saves model instance (weights and configuration) into EFF archive or . You can use "restore_from" method to fully restore instance from .nemo file. .nemo file is an archive (tar.gz) with the following: model_config.yaml - model configuration in .yaml format. You can deserialize this into cfg argument for model's constructor model_wights.chpt - model checkpoint Args: save_path: Path to .nemo file where model instance should be saved )path2yaml_file)filename source_dirN)tempfileTemporaryDirectoryrrr_MODEL_CONFIG_YAML_SPEAKER_MODELto_config_filer5 _VAD_MODELrRsave_tor`r_make_nemo_file_from_folder)rBr tmpdir config_yaml spkr_model vad_modelr#r#r$r*s    "zClusteringDiarizer.save_to restore_pathoverride_config_pathrIc Cst}t{}zktj||dt||dur"tj|t }n|}t |}tj tj|t r?tj|t |jj_n td|jdtj|t|jj_||_t |d||d}td|jd|dWt|nt|wWd|S1swY|S)N) path2file out_folderzModel zt does not contain a VAD model. A VAD model or manifest file withspeech segments need for diarization with this modelT)r,z was successfully restored from .)rgetcwdr$r%r_unpack_nemo_filechdirrrr&r loadrr)r3r7r8rrS__name__r'r>restore_map_location set_struct) rFr0r1rIcwdr,r-confinstancer#r#r$rPs0       zClusteringDiarizer.restore_fromreturncCs|jjSr )r2rrr#r#r$rszClusteringDiarizer.verboser )r)Nr)NN)!r9 __module__ __qualname____doc__rrrr. classmethodr&r;r=rqrtrfloatstrrrrr rrrr*rr]rQrPpropertyboolr __classcell__r#r#rDr$rAs8  # [(5N &)9rrpicklerrr$copyrtypingrrrrr]lightning.pytorch.utilitiesr omegaconfrr r nemo.collections.asr.metrics.derr 1nemo.collections.asr.models.classification_modelsr (nemo.collections.asr.models.label_modelsr (nemo.collections.asr.parts.mixins.mixinsr.nemo.collections.asr.parts.utils.speaker_utilsrrrrrrrr*nemo.collections.asr.parts.utils.vad_utilsrrrrnemo.core.classesr+nemo.core.connectors.save_restore_connectorr nemo.utilsrr__all__r&r)r'r+nnModulerr#r#r#r$s4       (