o %ݫi@sdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl Z ddlZddlZddlmZddlmZmZeeeZifddZd d Zifd d Z dI dJddZdIddZdIddZddZddZde ee e j!ffddZ"de ee e j!ffddZ#de ee e j!ffddZ$d d!Z%d"d#Z&Gd$d%d%Z'dId&d'Z(dKd(d)Z)dLd*d+Z*d,d-Z+d.d/Z,d0d1Z-d2d3Z.d4d5Z/d6d7Z0d8d9Z1dMd;d<Z2d=d>Z3dMd?d@Z4dNdCdDZ5dNdEdFZ6dGdHZ7dS)Ou Data reading and writing. Authors * Mirco Ravanelli 2020 * Aku Rouhe 2020 * Ju-Chieh Chou 2020 * Samuele Cornell 2020 * Abdel HEBA 2020 * Gaëlle Laperrière 2021 * Sahar Ghannay 2021 * Sylvain de Langen 2022 * Adel Moumen 2025 N)BytesIO)Union) get_logger)check_torchaudio_backendvalidate_backendcCsFt|ddd }t|}Wdn1swYt|||S)aLoads JSON and recursively formats string values. Arguments --------- json_path : str Path to CSV file. replacements : dict (Optional dict), e.g., {"data_folder": "/home/speechbrain/data"}. This is used to recursively format all string values in the data. Returns ------- dict JSON data with replacements applied. Example ------- >>> json_spec = '''{ ... "ex1": {"files": ["{ROOT}/mic1/ex1.wav", "{ROOT}/mic2/ex1.wav"], "id": 1}, ... "ex2": {"files": [{"spk1": "{ROOT}/ex2.wav"}, {"spk2": "{ROOT}/ex2.wav"}], "id": 2} ... } ... ''' >>> tmpfile = getfixture('tmpdir') / "test.json" >>> with open(tmpfile, "w", encoding="utf-8") as fo: ... _ = fo.write(json_spec) >>> data = load_data_json(tmpfile, {"ROOT": "/home"}) >>> data["ex1"]["files"][0] '/home/mic1/ex1.wav' >>> data["ex2"]["files"][1]["spk2"] '/home/ex2.wav' rutf-8encodingN)openjsonload_recursive_format) json_path replacementsfout_jsonrM/home/ubuntu/.local/lib/python3.10/site-packages/speechbrain/dataio/dataio.pyload_data_json(s !  rcCst|tr*|D] \}}t|tst|trt||q t|tr)||||<q t|trTt|D]"\}}t|tsAt|trGt||q3t|trS||||<q3dSdSN) isinstancedictitemslistrstr format_map enumerate)datarkeyitemirrrrOs      rc st|dddr}i}tj|dd}td}|D]X}z |d}|d=Wn ty.tdw||vr:td ||D]!\}} z|fd d | ||<Wq>ty_td | d wd|vrlt |d|d<|||<qWd|S1s|wY|S)aLoads CSV and formats string values. Uses the SpeechBrain legacy CSV data format, where the CSV must have an 'ID' field. If there is a field called duration, it is interpreted as a float. The rest of the fields are left as they are (legacy _format and _opts fields are not used to load the data in any special way). Bash-like string replacements with $to_replace are supported. Arguments --------- csv_path : str Path to CSV file. replacements : dict (Optional dict), e.g., {"data_folder": "/home/speechbrain/data"} This is used to recursively format all string values in the data. Returns ------- dict CSV data with replacements applied. Example ------- >>> csv_spec = '''ID,duration,wav_path ... utt1,1.45,$data_folder/utt1.wav ... utt2,2.0,$data_folder/utt2.wav ... ''' >>> tmpfile = getfixture("tmpdir") / "test.csv" >>> with open(tmpfile, "w", encoding="utf-8") as fo: ... _ = fo.write(csv_spec) >>> data = load_data_csv(tmpfile, {"data_folder": "/home"}) >>> data["utt1"]["wav_path"] '/home/utt1.wav' rnewliner T)skipinitialspacez \$([\w.]+)IDzBCSV has to have an 'ID' field, with unique ids for all data pointszDuplicate id: cst|dS)Nr)matchrrrszload_data_csv..z The item z/ requires replacements which were not supplied.durationN) r csv DictReaderrecompileKeyError ValueErrorrsubfloat) csv_pathrcsvfileresultreadervariable_finderrowdata_idrvaluerr*r load_data_csvdsB&        r=return'torchaudio.backend.common.AudioMetaDatacCstt|tj|\}}|dkrtj|d|d}ntj||d}|jdkr8tj|d|d\}}|d|_||_ |S) aRetrieves audio metadata from a file path. Behaves identically to torchaudio.info, but attempts to fix metadata (such as frame count) that is otherwise broken with certain torchaudio version and codec combinations. Note that this may cause full file traversal in certain cases! Arguments --------- path : str Path to the audio file to examine. backend : str, optional Audio backend to use for loading the audio file. Must be one of 'ffmpeg', 'sox', 'soundfile' or None. If None, uses torchaudio's default backend. Raises ------ ValueError If the `backend` is not one of the allowed values. Must be one of [None, 'ffmpeg', 'sox', 'soundfile']. Returns ------- torchaudio.backend.common.AudioMetaData Same value as returned by `torchaudio.info`, but may eventually have `num_frames` corrected if it otherwise would have been `== 0`. NOTE ---- Some codecs, such as MP3, require full file traversal for accurate length information to be retrieved. In these cases, you may as well read the entire audio file to avoid doubling the processing time. z.mp3mp3)formatbackendrBrF) normalizerBr') rospathsplitext torchaudioinfo num_framesr size sample_rate)rFrB _path_no_extpath_extrI channels_datarLrrrread_audio_infos$  rPc Cs t|t|dr|dt|tttfr.t|tr$t|}|dtj||d\}}nL|d}| dd}| d|}|dkrMt d|d|d ||kr\t d |d|d ||krp||}tj||||d \}}n tj|||d \}}| dd}| dS)a1General audio loading, based on a custom notation. Expected use case is in conjunction with Datasets specified by JSON. The parameter may just be a path to a file: `read_audio("/path/to/wav1.wav")` Alternatively, you can specify more options in a dict, e.g.: ``` # load a file from sample 8000 through 15999 read_audio({ "file": "/path/to/wav2.wav", "start": 8000, "stop": 16000 }) ``` Which codecs are supported depends on your torchaudio backend. Refer to `torchaudio.load` documentation for further details. Arguments --------- waveforms_obj : str, dict Path to audio or dict with the desired configuration. Keys for the dict variant: - `"file"` (str): Path to the audio file. - `"start"` (int, optional): The first sample to load. If unspecified, load from the very first frame. - `"stop"` (int, optional): The last sample to load (exclusive). If unspecified or equal to start, load from `start` to the end. Will not fail if `stop` is past the sample count of the file and will return less frames. backend : str, optional Audio backend to use for loading the audio file. Must be one of 'ffmpeg', 'sox', 'soundfile' or None. If None, uses torchaudio's default backend. Returns ------- torch.Tensor 1-channel: audio tensor with shape: `(samples, )`. >=2-channels: audio tensor with shape: `(samples, channels)`. Raises ------ ValueError If the `backend` is not one of the allowed values. Must be one of [None, 'ffmpeg', 'sox', 'soundfile']. Example ------- >>> dummywav = torch.rand(16000) >>> import os >>> tmpfile = str(getfixture('tmpdir') / "wave.wav") >>> write_audio(tmpfile, dummywav, 16000) >>> asr_example = { "wav": tmpfile, "spk_id": "foo", "words": "foo bar"} >>> loaded = read_audio(asr_example["wav"]) >>> loaded.allclose(dummywav.squeeze(0),atol=1e-4) # replace with eq with sox_io backend True seekrrCfilestartstopz"Invalid sample range (start < 0): z..!z%Invalid sample range (stop < start): z;! Hint: Omit "stop" if you want to read to the end of file.rJ frame_offsetrB)rWrBr') rhasattrrQrrrbytesrHr getr2 transposesqueeze) waveforms_objrBaudio_rFrSrTrJfsrrr read_audios:>         rac Cst|t|dr|dt|tttfr3t|tr$t|}|dtj||d\}}| ddS|d}t|t s?|g}g}| dd}| d|d}||}|D]} tj| |||d\}} | |qUt |d} | ddS) aGeneral audio loading, based on a custom notation. Expected use case is in conjunction with Datasets specified by JSON. The custom notation: The annotation can be just a path to a file: "/path/to/wav1.wav" Multiple (possibly multi-channel) files can be specified, as long as they have the same length: {"files": [ "/path/to/wav1.wav", "/path/to/wav2.wav" ] } Or you can specify a single file more succinctly: {"files": "/path/to/wav2.wav"} Offset number samples and stop number samples also can be specified to read only a segment within the files. {"files": [ "/path/to/wav1.wav", "/path/to/wav2.wav" ] "start": 8000 "stop": 16000 } Arguments --------- waveforms_obj : str, dict Audio reading annotation, see above for format. backend : str, optional Audio backend to use for loading the audio file. Must be one of 'ffmpeg', 'sox', 'soundfile' or None. If None, uses torchaudio's default backend. Raises ------ ValueError If the `backend` is not one of the allowed values. Must be one of [None, 'ffmpeg', 'sox', 'soundfile']. Returns ------- torch.Tensor Audio tensor with shape: (samples, ). Example ------- >>> dummywav = torch.rand(16000, 2) >>> import os >>> tmpfile = str(getfixture('tmpdir') / "wave.wav") >>> write_audio(tmpfile, dummywav, 16000) >>> asr_example = { "wav": tmpfile, "spk_id": "foo", "words": "foo bar"} >>> loaded = read_audio(asr_example["wav"]) >>> loaded.allclose(dummywav.squeeze(0),atol=1e-4) # replace with eq with sox_io backend True rQrrCr'filesrSrTrV)rrXrQrrrrYrHr r[rrZappendtorchcat) r]rBr^r_rb waveformsrSrTrJrr`outrrrread_audio_multichanneles.>          rhcCsFt|jdkr|dd}n t|jdkr|d}t|||dS)aWrite audio on disk. It is basically a wrapper to support saving audio signals in the speechbrain format (audio, channels). Arguments --------- filepath: path Path where to save the audio file. audio : torch.Tensor Audio file in the expected speechbrain format (signal, channels). samplerate: int Sample rate (e.g., 16000). Example ------- >>> import os >>> tmpfile = str(getfixture('tmpdir') / "wave.wav") >>> dummywav = torch.rand(16000, 2) >>> write_audio(tmpfile, dummywav, 16000) >>> loaded = read_audio(tmpfile) >>> loaded.allclose(dummywav,atol=1e-4) # replace with eq with sox_io backend True rr'N)lenshaper[ unsqueezerHsave)filepathr^ sampleraterrr write_audios  rpcCs:t|d}t|}Wd|S1swY|S)zUtility function for loading .pkl pickle files. Arguments --------- pickle_path : str Path to pickle file. Returns ------- out : object Python object loaded from pickle. rbN)r pickler ) pickle_pathrrgrrr load_pickles  rtxcC>t|tjr |St|tjrt|Stj|tjdS)z Arguments --------- x : (list, tuple, np.ndarray) Input data to be converted to torch float. Returns ------- tensor : torch.Tensor Data now in torch.tensor float datatype. dtype)rrdTensorr4npndarray from_numpytensorrurrrto_floatTensor  rcCrv)z Arguments --------- x : (list, tuple, np.ndarray) Input data to be converted to torch double. Returns ------- tensor : torch.Tensor Data now in torch.tensor double datatype. rw)rrdrydoublerzr{r|r}r~rrrto_doubleTensor rrcCrv)z Arguments --------- x : (list, tuple, np.ndarray) Input data to be converted to torch long. Returns ------- tensor : torch.Tensor Data now in torch.tensor long datatype. rw)rrdrylongrzr{r|r}r~rrr to_longTensor!rrcsfdd|DS)a Convert a batch of integer IDs to string labels. Arguments --------- batch : list List of lists, a batch of sequences. ind2lab : dict Mapping from integer IDs to labels. Returns ------- list List of lists, same size as batch, with labels from ind2lab. Example ------- >>> ind2lab = {1: "h", 2: "e", 3: "l", 4: "o"} >>> out = convert_index_to_lab([[4,1], [1,2,3,3,4]], ind2lab) >>> for seq in out: ... print("".join(seq)) oh hello csg|] }fdd|DqS)csg|]}t|qSr)int).0indexind2labrr Msz3convert_index_to_lab...r)rseqrrrrMsz(convert_index_to_lab..r)batchrrrrconvert_index_to_lab5srcCs |jd}t|||}|S)aConverts SpeechBrain style relative length to the absolute duration. Operates on batch level. Arguments --------- batch : torch.Tensor Sequences to determine the duration for. relative_lens : torch.Tensor The relative length of each sequence in batch. The longest sequence in the batch needs to have relative length 1.0. rate : float The rate at which sequence elements occur in real-world time. Sample rate, if batch is raw wavs (recommended) or 1/frame_shift if batch is features. This has to have 1/s as the unit. Returns ------- torch.Tensor Duration of each sequence in seconds. Example ------- >>> batch = torch.ones(2, 16000) >>> relative_lens = torch.tensor([3./4., 1.0]) >>> rate = 16000 >>> print(relative_time_to_absolute(batch, relative_lens, rate)) tensor([0.7500, 1.0000]) r')rkrdround)r relative_lensratemax_len durationsrrrrelative_time_to_absolutePs rc@s@eZdZdZifddZddZddZdd Zed d Z d S) IterativeCSVWritera%Write CSV files a line at a time. Arguments --------- outstream : file-object A writeable stream data_fields : list List of the optional keys to write. Each key will be expanded to the SpeechBrain format, producing three fields: key, key_format, key_opts. defaults : dict Mapping from CSV key to corresponding default value. Example ------- >>> import io >>> f = io.StringIO() >>> writer = IterativeCSVWriter(f, ["phn"]) >>> print(f.getvalue()) ID,duration,phn,phn_format,phn_opts >>> writer.write("UTT1",2.5,"sil hh ee ll ll oo sil","string","") >>> print(f.getvalue()) ID,duration,phn,phn_format,phn_opts UTT1,2.5,sil hh ee ll ll oo sil,string, >>> writer.write(ID="UTT2",phn="sil ww oo rr ll dd sil",phn_format="string") >>> print(f.getvalue()) ID,duration,phn,phn_format,phn_opts UTT1,2.5,sil hh ee ll ll oo sil,string, UTT2,,sil ww oo rr ll dd sil,string, >>> writer.set_default('phn_format', 'string') >>> writer.write_batch(ID=["UTT3","UTT4"],phn=["ff oo oo", "bb aa rr"]) >>> print(f.getvalue()) ID,duration,phn,phn_format,phn_opts UTT1,2.5,sil hh ee ll ll oo sil,string, UTT2,,sil ww oo rr ll dd sil,string, UTT3,,ff oo oo,string, UTT4,,bb aa rr,string, cCs8||_ddg|||_||_|jd|jdS)Nr&r,,) _outstream_expand_data_fieldsfieldsdefaultswritejoin)self outstream data_fieldsrrrr__init__szIterativeCSVWriter.__init__cCs&||jvr t|d||j|<dS)zSets a default value for the given CSV field. Arguments --------- field : str A field in the CSV. value : str The default value. z is not a field in this CSV!N)rr2r)rfieldr<rrr set_defaults zIterativeCSVWriter.set_defaultcs|r|rtd|rt|t|jkrtddd|D}|rWrites one data line into the CSV. Arguments --------- *args : tuple Supply every field with a value in positional form OR. **kwargs : dict Supply certain fields by key. The ID field is mandatory for all lines, but others can be left empty. ;Use either positional fields or named fields, but not both.Need consistent fieldscSsg|]}t|qSrr()rargrrrrsz,IterativeCSVWriter.write..r&I'll need to see some IDcsg|] }t|dqS)r")rrZ)rr full_valsrrrs rN) r2rjrrcopyupdaterrr)rargskwargsto_writerrrrs     zIterativeCSVWriter.writecOs|r|rtd|r#t|t|jkrtdt|D]}|j|q|rId|vr-td|}t|D]}tt||}|jdi|q7dSdS)aWrites a batch of lines into the CSV. Here each argument should be a list with the same length. Arguments --------- *args : tuple Supply every field with a value in positional form OR. **kwargs : dict Supply certain fields by key. The ID field is mandatory for all lines, but others can be left empty. rrr&rNr)r2rjrziprkeysvaluesr)rrrarg_rowr value_row kwarg_rowrrr write_batchs$   zIterativeCSVWriter.write_batchcCs8g}|D]}||||d||dq|S)N_format_opts)rc)rexpanded data_fieldrrrrs  z&IterativeCSVWriter._expand_data_fieldsN) __name__ __module__ __qualname____doc__rrrr staticmethodrrrrrrss &rcCs~tjtj|ddt|ddd@}t|tjr|}t|t j r(|}t|t r8|D]}t ||dq/t|t rKt ||dWddSWddS1sVwYdS)aWrite data in text format. Arguments --------- data : str, list, torch.Tensor, numpy.ndarray The data to write in the text file. filename : str Path to file where to write the data. sampling_rate : None Not used, just here for interface compatibility. Example ------- >>> tmpdir = getfixture('tmpdir') >>> signal=torch.tensor([1,2,3,4]) >>> write_txt_file(signal, tmpdir / 'example.txt') T)exist_okwrr )rRN)rEmakedirsrFdirnamer rrdrytolistrzr{rprintr)rfilename sampling_ratefoutlinerrrwrite_txt_files    "rcCs^t|tjr |}t|tjr|}t|tr"|D]}t|qt|tr-t|dSdS)aWrite data to standard output. Arguments --------- data : str, list, torch.Tensor, numpy.ndarray The data to write in the text file. filename : None Not used, just here for compatibility. sampling_rate : None Not used, just here for compatibility. Example ------- >>> tmpdir = getfixture('tmpdir') >>> signal = torch.tensor([[1,2,3,4]]) >>> write_stdout(signal, tmpdir / 'example.txt') [1, 2, 3, 4] N) rrdryrrzr{rrr)rrrrrrr write_stdouts      rcCst|jdks J|dur|}tj||j|jd t||| dk}|dur1|j}|dur8|j}tj |||d}|S)a Creates a binary mask for each sequence. Reference: https://discuss.pytorch.org/t/how-to-generate-variable-length-mask/23397/3 Arguments --------- length : torch.LongTensor Containing the length of each sequence in the batch. Must be 1D. max_len : int Max length for the mask, also the size of the second dimension. dtype : torch.dtype, default: None The dtype of the generated mask. device: torch.device, default: None The device to put the mask variable. Returns ------- mask : tensor The binary mask. Example ------- >>> length=torch.Tensor([1,2,3]) >>> mask=length_to_mask(length) >>> mask tensor([[1., 0., 0.], [1., 1., 0.], [1., 1., 1.]]) r'N)devicerx)rxr) rjrkmaxrr rdarangerrxexpandrl as_tensor)lengthrrxrmaskrrrlength_to_mask3s rcCsTzddl}Wn tytdwdd|d|d|d|d D}|S) aRead labels in kaldi format. Uses kaldi IO. Arguments --------- kaldi_ali : str Path to directory where kaldi alignments are stored. kaldi_lab_opts : str A string that contains the options for reading the kaldi alignments. Returns ------- lab : dict A dictionary containing the labels. Note ---- This depends on kaldi-io-for-python. Install it separately. See: https://github.com/vesis84/kaldi-io-for-python Example ------- This example requires kaldi files. ``` lab_folder = '/home/kaldi/egs/TIMIT/s5/exp/dnn4_pretrain-dbn_dnn_ali' read_kaldi_lab(lab_folder, 'ali-to-pdf') ``` rNz2Could not import kaldi_io. Install it to use this.cSsi|]\}}||qSrr)rkvrrr sz"read_kaldi_lab..z gunzip -c z /ali*.gz |  z/final.mdl ark:- ark:-|)kaldi_io ImportErrorread_vec_int_ark) kaldi_alikaldi_lab_optsrlabrrrread_kaldi_labcs.   rcCsbd}t}t|d} ||}|sn||q Wd|S1s(wY|S)aOGet the md5 checksum of an input file. Arguments --------- file : str Path to file for which compute the checksum. Returns ------- md5 Checksum for the given filepath. Example ------- >>> get_md5('tests/samples/single-mic/example1.wav') 'c482d0081ca35302d30d12f1136c34e5' irqTN)hashlibmd5r readr hexdigest)rRBUF_SIZErrrrrrget_md5s    rcCs(i}|D]}t|||<qt||dS)aSaves the md5 of a list of input files as a pickled dict into a file. Arguments --------- files : list List of input files from which we will compute the md5. out_file : str The path where to store the output pkl file. Example ------- >>> files = ['tests/samples/single-mic/example1.wav'] >>> tmpdir = getfixture('tmpdir') >>> save_md5(files, tmpdir / "md5.pkl") N)rsave_pkl)rbout_filemd5_dictrRrrrsave_md5srcCs<t|d}t||WddS1swYdS)aMSave an object in pkl format. Arguments --------- obj : object Object to save in pkl format file : str Path to the output file Example ------- >>> tmpfile = getfixture('tmpdir') / "example.pkl" >>> save_pkl([1, 2, 3, 4, 5], tmpfile) >>> load_pkl(tmpfile) [1, 2, 3, 4, 5] wbN)r rrdump)objrRrrrrrs "rc Csd}|dkrtj|drtd|d8}nn|dkszJt|ddddt|d}t|Wd Wtj|drKt |dSS1sOwYWtj|drft |dd Sd Stj|drxt |dww) zLoads a pkl file. For an example, see `save_pkl`. Arguments --------- file : str Path to the input pkl file. Returns ------- The loaded object. drz.lockr'rrr rqN) rErFisfiletimesleepr closerrr remove)rRcountrrrrload_pkls*   rcCs>|}|jd}||d|}tj||gdd}|S)aICreate labels with token at the beginning. Arguments --------- label : torch.IntTensor Containing the original labels. Must be of size: [batch_size, max_length]. bos_index : int The index for token. Returns ------- new_label : tensor The new label with at the beginning. Example ------- >>> label=torch.LongTensor([[1,0,0], [2,3,0], [4,5,6]]) >>> new_label=prepend_bos_token(label, bos_index=7) >>> new_label tensor([[7, 1, 0, 0], [7, 2, 3, 0], [7, 4, 5, 6]]) rr'dim)rclonerk new_zerosfill_rdre)label bos_index new_label batch_sizebosrrrprepend_bos_tokens  rcCsN|}|jd}||d}tj||gdd}||t||f<|S)aCreate labels with token appended. Arguments --------- label : torch.IntTensor Containing the original labels. Must be of size: [batch_size, max_length] length : torch.LongTensor Containing the original length of each label sequences. Must be 1D. eos_index : int The index for token. Returns ------- new_label : tensor The new label with appended. Example ------- >>> label=torch.IntTensor([[1,0,0], [2,3,0], [4,5,6]]) >>> length=torch.LongTensor([1,2,3]) >>> new_label=append_eos_token(label, length, eos_index=7) >>> new_label tensor([[1, 7, 0, 0], [2, 3, 7, 0], [4, 5, 6, 7]], dtype=torch.int32) rr'r)rrrkrrdrerr)rr eos_indexrrpadrrrappend_eos_token%s   rr_cCs,g}|D]}d||}||q|S)aMerge characters sequences into word sequences. Arguments --------- sequences : list Each item contains a list, and this list contains a character sequence. space : string The token represents space. Default: _ Returns ------- The list contains word sequences for each sentence. Example ------- >>> sequences = [["a", "b", "_", "c", "_", "d", "e"], ["e", "f", "g", "_", "h", "i"]] >>> results = merge_char(sequences) >>> results [['ab', 'c', 'de'], ['efg', 'hi']] r")rsplitrc) sequencesspaceresultsrwordsrrr merge_charIs  rc CsVtj||}tj|rtdttj||dddd }|}Wdn1s/wYg}|D]B}ttj||ddd,}t|D]\}} |dkre| |krdt d|ddt d qK| | qKWdn1suwYq8t|d dd }| ||D]} | | qWdn1swYt|d dS) a4Merging several csv files into one file. Arguments --------- data_folder : string The folder to store csv files to be merged and after merging. csv_lst : list Filenames of csv file to be merged. merged_csv : string The filename to write the merged csv file. Example ------- >>> tmpdir = getfixture('tmpdir') >>> os.symlink(os.path.realpath("tests/samples/annotation/speech.csv"), tmpdir / "speech.csv") >>> merge_csvs(tmpdir, ... ["speech.csv", "speech.csv"], ... "test_csv_merge.csv") z,Skipping merging. Completed in previous run.rr"rr#NzDifferent header for z and .rr z is created.) rErFrrloggerrIr readlinerr2r-rcr) data_foldercsv_lst merged_csv write_pathrheaderlinescsv_filer!rrrr merge_csvsesB      rcCs*g}|D]}t||}||q|S)aSplit word sequences into character sequences. Arguments --------- sequences: list Each item contains a list, and this list contains a words sequence. space: string The token represents space. Default: _ Returns ------- The list contains word sequences for each sentence. Example ------- >>> sequences = [['ab', 'c', 'de'], ['efg', 'hi']] >>> results = split_word(sequences) >>> results [['a', 'b', '_', 'c', '_', 'd', 'e'], ['e', 'f', 'g', '_', 'h', 'i']] )rrrc)rrrrcharsrrr split_words  rr'cCsT||}t|||}|dd|d}|d||}|||<dS)aSets the value of any padding on the specified tensor to mask_value. For instance, this can be used to zero out the outputs of an autoencoder during training past the specified length. This is an in-place operation Arguments --------- tensor: torch.Tensor a tensor of arbitrary dimension length: torch.Tensor a 1-D tensor of lengths len_dim: int the dimension representing the length mask_value: mixed the value to be assigned to padding positions Example ------- >>> import torch >>> x = torch.arange(5).unsqueeze(0).repeat(3, 1) >>> x = x + torch.arange(3).unsqueeze(-1) >>> x tensor([[0, 1, 2, 3, 4], [1, 2, 3, 4, 5], [2, 3, 4, 5, 6]]) >>> length = torch.tensor([0.4, 1.0, 0.6]) >>> clean_padding_(x, length=length, mask_value=10.) >>> x tensor([[ 0, 1, 10, 10, 10], [ 1, 2, 3, 4, 5], [ 2, 3, 4, 10, 10]]) >>> x = torch.arange(5)[None, :, None].repeat(3, 1, 2) >>> x = x + torch.arange(3)[:, None, None] >>> x = x * torch.arange(1, 3)[None, None, :] >>> x = x.transpose(1, 2) >>> x tensor([[[ 0, 1, 2, 3, 4], [ 0, 2, 4, 6, 8]], [[ 1, 2, 3, 4, 5], [ 2, 4, 6, 8, 10]], [[ 2, 3, 4, 5, 6], [ 4, 6, 8, 10, 12]]]) >>> clean_padding_(x, length=length, mask_value=10., len_dim=2) >>> x tensor([[[ 0, 1, 10, 10, 10], [ 0, 2, 10, 10, 10]], [[ 1, 2, 3, 4, 5], [ 2, 4, 6, 8, 10]], [[ 2, 3, 4, 10, 10], [ 4, 6, 8, 10, 10]]]) ).rrir'N)rKrboolrr[ expand_as)r}rlen_dim mask_valuerr mask_unsqmask_trrrclean_padding_s :rcCs|}t|||||S)a_Sets the value of any padding on the specified tensor to mask_value. For instance, this can be used to zero out the outputs of an autoencoder during training past the specified length. This version of the operation does not modify the original tensor Arguments --------- tensor: torch.Tensor a tensor of arbitrary dimension length: torch.Tensor a 1-D tensor of lengths len_dim: int the dimension representing the length mask_value: mixed the value to be assigned to padding positions Returns ------- result: torch.Tensor Tensor with updated padding. Example ------- >>> import torch >>> x = torch.arange(5).unsqueeze(0).repeat(3, 1) >>> x = x + torch.arange(3).unsqueeze(-1) >>> x tensor([[0, 1, 2, 3, 4], [1, 2, 3, 4, 5], [2, 3, 4, 5, 6]]) >>> length = torch.tensor([0.4, 1.0, 0.6]) >>> x_p = clean_padding(x, length=length, mask_value=10.) >>> x_p tensor([[ 0, 1, 10, 10, 10], [ 1, 2, 3, 4, 5], [ 2, 3, 4, 10, 10]]) >>> x = torch.arange(5)[None, :, None].repeat(3, 1, 2) >>> x = x + torch.arange(3)[:, None, None] >>> x = x * torch.arange(1, 3)[None, None, :] >>> x = x.transpose(1, 2) >>> x tensor([[[ 0, 1, 2, 3, 4], [ 0, 2, 4, 6, 8]], [[ 1, 2, 3, 4, 5], [ 2, 4, 6, 8, 10]], [[ 2, 3, 4, 5, 6], [ 4, 6, 8, 10, 12]]]) >>> x_p = clean_padding(x, length=length, mask_value=10., len_dim=2) >>> x_p tensor([[[ 0, 1, 10, 10, 10], [ 0, 2, 10, 10, 10]], [[ 1, 2, 3, 4, 5], [ 2, 4, 6, 8, 10]], [[ 2, 3, 4, 10, 10], [ 4, 6, 8, 10, 10]]]) )rr)r}rrrr7rrr clean_paddings@rc Cs@g}|D]}d|}||}g}g}d} d} |D]X} t|| rJ| r<|r>> sequences = [['','_','n','o','_','>','_','','_','L','e','_','M','a','n','s','_','>'], ['','_','s','i','_','>'],['v','a','_','b','e','n','e']] >>> results = extract_concepts_values(sequences, True, '<', '>', '_') >>> results [[' no', ' Le Mans'], [' si'], ['']] r"FrrT)rrr/r)rjrc) r keep_valuestag_intag_outrrsequenceprocessed_sequencer<kept concept_openwordrrrextract_concepts_values7sR              r$r)r>r?)NN)NNN)r_)r'r)8rr-rr rErrr/riortypingrnumpyrzrdrHspeechbrain.utils.loggerr%speechbrain.utils.torch_audio_backendrrrrrrr=rPrarhrprtrtupler{rrrrrrrrrrrrrrrrrrrrrr$rrrrsd    ' I H qb #  ! 03" $ 0  A E