o Si@sddlZddlZddlmZddlmZmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZmZmZmZddlZddlZddlmZddlmZmZdd lmZm Z m!Z!dd l"m#Z#dd l$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6m7Z7m8Z8ddl9m:Z:ddl;mZ>ddl?m@Z@ddlAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPeGdddZQeGddde0ZRdS)N) dataclass)partialreduce)BytesIO)add) AnyCallableDict GeneratorIterableListLiteralOptionalTupleUnion) IntervalTree)Array TemporalArray) Recording VideoInfo%get_audio_duration_mismatch_tolerance) save_audio) AudioMixer VideoMixer audio_energy)AudioTransform AugmentFnLoudnessNormalizationReverbWithImpulseResponse)Codec)Cut)DataCut) PaddingCut)FeatureExtractor FeatureMixer create_default_feature_extractor)Features)FeaturesWriter)Image)SupervisionSegment)DEFAULT_PADDING_VALUE LOG_EPSILONDecibelsPathlikeSeconds add_durationscompute_num_framescompute_num_samplesfastcopyhash_str_to_intmerge_items_with_delimiteroverlapsperturb_num_samplesrich_exception_infouuid4c@sreZdZUdZeeefed<dZe ed<dZ e ed<dZ e eed<dd Zed efd d Zd efddZdS)MixTrackz Represents a single track in a mix of Cuts. Points to a specific DataCut or PaddingCut and holds information on how to mix it with other Cuts, relative to the first track in a mix. cutNtypeoffsetsnrcCst|jj|_dSN)r;r:__name__selfrCD/home/ubuntu/.local/lib/python3.10/site-packages/lhotse/cut/mixed.py __post_init__NszMixTrack.__post_init__datacCs8ddlm}|d}|d|d<t||fi|S)N)deserialize_cutr:r;)setrHpopr9)rFrHcut_dictrCrCrD from_dictQs  zMixTrack.from_dictreturncCs.|j|j|jd}|jdur|j|d<|S)N)r:r;r=r>)r:to_dictr;r=r>rBansrCrCrDrNZs  zMixTrack.to_dict)r@ __module__ __qualname____doc__rr!r"__annotations__r;strr=r.r>rr,rE staticmethoddictrLr rNrCrCrCrDr9Bs   r9csReZdZUdZeed<eeed<dZe ee ed<e dee fddZ e defd d Ze defd d Ze deeeeffd dZe defddZe defddZe defddZe defddZdedefddZe de efddZe de efddZe de efddZe de efd d!Ze de efd"d#Ze de efd$d%Ze de efd&d'Z de!fd(d)Z"de#e$eee%e&e'e(e)ffddffd*d+Z*d,ed-e+ddffd.d/ Z,d0ede+fd1d2Z-d0edefd3d4Z.d0ede/j0fd5d6Z1d7ede$ee2ffd8d9Z3 : ; ; ;dded?eddf d@dAZ4 :ddBeddCfdDdEZ5dFdd;dGddHdIedJe edKedLedMe e6ee7fde8f dNdOZ9dPdGd;dQdJedRedLedSeddf dTdUZ:ddde;dVdGdfdJedWedXedYe b c Gddde?dee>> cut = MixedCut(...) >>> mono_features = cut.load_features() >>> assert len(mono_features.shape) == 2 >>> multi_features = cut.load_features(mixed=False) >>> # Now, the first dimension is the channel. >>> assert len(multi_features.shape) == 3 .. note:: MixedCut is different from MultiCut, which is intended to represent multi-channel recordings that share the same supervisions. .. note:: Each track in a MixedCut can be either a MonoCut, MultiCut, or PaddingCut. .. note:: The ``transforms`` field is a list of dictionaries that describe the transformations that should be applied to the track after mixing. See also: - :class:`lhotse.cut.Cut` - :class:`lhotse.cut.MonoCut` - :class:`lhotse.cut.MultiCut` - :class:`lhotse.cut.CutSet` idtracksN transformsrMcCsdd|jDS)z Lists the supervisions of the underlying source cuts. Each segment start time will be adjusted by the track offset. cSs&g|]}|jjD]}||jqqSrC)r: supervisions with_offsetr=).0tracksegmentrCrCrD s z)MixedCut.supervisions..rZrArCrCrDr\szMixedCut.supervisionscCsdSNrrCrArCrCrDstartszMixedCut.startcCs dd|jD}tt|ddS)Ncss|] }|j|jjVqdSr?)r=r:durationr^r_rCrCrD z$MixedCut.duration..ndigits)rZroundmax)rBtrack_durationsrCrCrDreszMixedCut.durationcCs|j}|dkr tt|SdS)NrGr) num_channelslistrange)rBrorCrCrDchannelszMixedCut.channelcC|jjSr?)_first_non_padding_cut has_featuresrArCrCrDruzMixedCut.has_featurescCrsr?)rt has_recordingrArCrCrDrwrvzMixedCut.has_recordingcCrsr?)rt has_videorArCrCrDrxrvzMixedCut.has_videocCtdd|jDS)Ncs|]}|jjVqdSr?)r: is_in_memoryrfrCrCrDrgz(MixedCut.is_in_memory..)anyrZrArCrCrDr{zMixedCut.is_in_memoryfieldcCs |j|Sr?)rthas)rBrrCrCrDrs z MixedCut.hascCs|jr t|j|j|jdSdS)N)re frame_shift sampling_rate)rur0rerrrArCrCrD num_framesszMixedCut.num_framescC|jdjjSrc)rZr:rrArCrCrDrzMixedCut.frame_shiftcCrrc)rZr:rrArCrCrDrrzMixedCut.sampling_ratecCst|j|jSr?)r1rerrArCrCrD num_samplesrzMixedCut.num_samplescCrrc)rZr: num_featuresrArCrCrDrrzMixedCut.num_featurescCry)Ncsrzr?)r:rorfrCrCrDrgr|z(MixedCut.num_channels..)rmrZrArCrCrDroszMixedCut.num_channelscCs|jr|jjjSdSr?)rurtfeaturesr;rArCrCrD features_typer~zMixedCut.features_typecCs>|jdd|jDt|jd}|jrdd|jD|d<|S)NcSg|]}|qSrCrNr^trCrCrDraz$MixedCut.to_dict..)rYrZr;cSrrCrrrCrCrDrarr[)rYrZr;r@r[rOrCrCrDrNszMixedCut.to_dictcCs |jS)a Iterate over each data piece attached to this cut. Returns a generator yielding tuples of ``(key, manifest)``, where ``key`` is the name of the attribute under which ``manifest`` is found. ``manifest`` is of type :class:`~lhotse.Recording`, :class:`~lhotse.Features`, :class:`~lhotse.TemporalArray`, :class:`~lhotse.Array`, or :class:`~lhotse.Image`. For example, if ``key`` is ``recording``, then ``manifest`` is ``self.recording``. )rt iter_datarArCrCrDrs zMixedCut.iter_datakeyvaluecs.||jvrt||dSt|j||dS)a This magic function is called when the user tries to set an attribute. We use it as syntactic sugar to store custom attributes in ``self._first_non_padding_cut.custom`` field, so that they can be (de)serialized later. Setting a ``None`` value will remove the attribute from ``custom``. .. note:: MixedCut doesn't have its own ``custom`` field, and by convention always refers to the ``custom`` field on its first non padding cut. N)__dataclass_fields__super __setattr__setattrrt)rBrr __class__rCrDrs zMixedCut.__setattr__namecCs|drt|dr|dd}t|j|S|dkr2i}|jD] }|jj}r/||q"|Sz ||\}}t ||WSt yNtd|dw)a This magic function is called when the user tries to access an attribute of :class:`.MixedCut` that doesn't exist. It is used for accessing the custom attributes of cuts. We support exactly one scenario for mixed cuts: If :attr:`tracks` contains exactly one :class:`.MonoCut` object (and an arbitrary number of :class:`.PaddingCut` objects), we will look up the custom attributes of that cut. If one of the custom attributes is of type :class:`~lhotse.array.Array` or :class:`~lhotse.array.TemporalArray` we'll also support loading those arrays (see example below). Additionally, we will incorporate extra padding as dictated by padding cuts. Example: >>> cut = MonoCut('cut1', start=0, duration=4, channel=0) >>> cut.alignment = TemporalArray(...) >>> mixed_cut = cut.pad(10, pad_value_dict={'alignment': -1}) >>> ali = mixed_cut.load_alignment() __load_NcustomzNo such attribute: 'zt' (note: custom attributes are not supported when a MixedCut consists of more than one MonoCut with that attribute).) startswithAttributeErrorr load_customrZr:rupdate=_assert_one_data_cut_with_attr_and_return_it_with_track_indexgetattrAssertionError)rBr attr_namerPrcstmnon_padding_idxmono_cutrCrCrD __getattr__s.          zMixedCut.__getattr__cCs||\}}t||Sr?)rhasattr)rBrrrrCrCrD has_custom?s  zMixedCut.has_customc Csddlm}m}||\}}t||}t||r||S|j|j}|j }t|t r?| j |j |ddj |dd S||} zdd|jDdj} | |} Wnt} Y|| |j|j||| dS) a Load custom data as numpy array. The custom data is expected to have been stored in cuts ``custom`` field as an :class:`~lhotse.array.Array` or :class:`~lhotse.array.TemporalArray` manifest. .. note:: It works with Array manifests stored via attribute assignments, e.g.: ``cut.my_custom_data = Array(...)``. .. warning:: For :class:`.MixedCut`, this will only work if the mixed cut consists of a single :class:`.MonoCut` and an arbitrary number of :class:`.PaddingCuts`. This is because it is generally undefined how to mix arbitrary arrays. :param name: name of the custom attribute. :return: a numpy array with the data (after padding). r)r pad_arrayleft)re directionrightcSsg|] }t|jtr|jqSrC isinstancer:r"rrCrCrDra|s  z(MixedCut.load_custom..) temporal_dimrr=padded_duration pad_value) lhotse.arrayrrrrrrrZr=rerto_cutpad load_audiorr*rr) rBrrrrrmanifest left_paddingrarraypad_value_dictrrCrCrDrGsL        zMixedCut.load_customrcsdddt|jD}fdd|D}t|dks(Jdt|dd||d\}}||fS) NcSs&g|]\}}t|jtr||jfqSrCrr:r!r^idxrrCrCrDras  zZMixedCut._assert_one_data_cut_with_attr_and_return_it_with_track_index..cs,g|]\}}|jdur|jvr||fqSr?r)r^rr:rrCrDras rGzThis MixedCut has z+ non-padding cuts with a custom attribute 'z'. We currently don't support mixing custom attributes. Consider dropping the attribute on all but one of DataCuts. Problematic cut: r) enumeraterZlen)rBrnon_padding_cuts!non_padding_cuts_with_custom_attrrrrCrrDrs   zFMixedCut._assert_one_data_cut_with_attr_and_return_it_with_track_indexflacT audio_formatr load_featuresrcs"t|fdd|jDdS)a Load data (audio, features, or custom arrays) into memory and attach them to a copy of the manifest. This is useful when you want to store cuts together with the actual data in some binary format that enables sequential data reads. Audio is encoded with ``audio_format`` (compatible with ``torchaudio.save``), floating point features are encoded with lilcom, and other arrays are pickled. c s(g|]}t||jjddqS))rrrrr:)r2r:move_to_memoryrrrrrrCrDras z+MixedCut.move_to_memory..rbr2rZ)rBrrrrrCrrDrs  zMixedCut.move_to_memoryencodingr cKs`|jdd}t}t|||j|dtj||jd}t| dd|j D|j dj j dS) a Convert this MixedCut to a MonoCut by mixing all tracks and channels into a single one. The result audio array is stored in memory, and can be saved to disk by calling ``cut.save_audio(path, ...)`` on the result. .. hint:: the resulting MonoCut will have ``custom`` field populated with the ``custom`` value from the first track of the MixedCut. :param encoding: any of "wav", "flac", or "opus". :return: a new MonoCut instance. T) mono_downmix)format recording_idcSsg|]}t|ddqS)r)rr)r2r^srCrCrDrasz$MixedCut.to_mono..r)r\r)rrrrr from_bytesgetvaluerYr2rr\rZr:r)rBrkwargssamplesstream recordingrCrCrDto_monos  zMixedCut.to_monor<Fr=rekeep_excessive_supervisions preserve_id_supervisions_indexr=rerrrc Cs|dks Jd|dg}|j}|durt|| |jdnt|||jd}t|jdddD]o} tt|| j |jdd} tt| j| |jdd} t| j| jj|jd} | |kr[q/d} | |krx|durot| | |jd} n t| | |jd} t| jj| | |jd}|dkrq/|t | jj | ||||d | | j d q/t d d |Ddkrt |r|jntt||jd t||jdSt |dkr|djSt|r|jntt|d}tdd|jDrdd t|jDd}t|j|dd|j|<|S)a% Returns a new MixedCut that is a sub-region of the current MixedCut. This method truncates the underlying Cuts and modifies their offsets in the mix, as needed. Tracks that do not fit in the truncated cut are removed. Note that no operation is done on the actual features - it's only during the call to load_features() when the actual changes happen (a subset of features is loaded). :param offset: float (seconds), controls the start of the new cut relative to the current MixedCut's start. :param duration: optional float (seconds), controls the duration of the resulting MixedCut. By default, the duration is (end of the cut before truncation) - (offset). :param keep_excessive_supervisions: bool. Since trimming may happen inside a SupervisionSegment, the caller has an option to either keep or discard such supervisions. :param preserve_id: bool. Should the truncated cut keep the same ID or get a new, random one. :return: a new MixedCut instance. rz3Offset for truncate must be non-negative (provided ).NrcS|jSr?)r=)rrCrCrDz#MixedCut.truncate..rrr:r=r>cSg|] }t|jts|qSrCrrrCrCrDraEz%MixedCut.truncate..r<)rYrer feat_valuerrGrYrZcss&|]}|jdupt|jtVqdSr?)r>rr:r"rrCrCrDrg\s z$MixedCut.truncate..cSs g|] \}}t|jts|qSrCrrrCrCrDra_s  )r>)rer/rsortedrZrmr=r:appendr9truncater>rr"rYrUr8r1rXallrr2)rBr=rerrr new_tracks old_duration new_mix_endr_ cut_offset track_offset track_endcut_duration_decrease new_durationnew_cutfirst_non_padding_track_idxrCrCrDrs          zMixedCut.truncateboth)rr pad_silencerrcCstd)a^ This raises a ValueError since extending a MixedCut is not defined. :param duration: float (seconds), duration (in seconds) to extend the MixedCut. :param direction: string, 'left', 'right' or 'both'. Determines whether to extend on the left, right, or both sides. If 'both', extend on both sides by the duration specified in `duration`. :param preserve_id: bool. Should the extended cut keep the same ID or get a new, random one. :param pad_silence: bool. See usage in `lhotse.cut.MonoCut.extend_by`. :return: a new MixedCut instance. z5The extend_by() method is not defined for a MixedCut.) ValueError)rBrerrrrCrCrD extend_byjszMixedCut.extend_byrrrpad_feat_valuerc Cs$ddlm}|||||||||dS)a Return a new MixedCut, padded with zeros in the recording, and ``pad_feat_value`` in each feature bin. The user can choose to pad either to a specific `duration`; a specific number of frames `num_frames`; or a specific number of samples `num_samples`. The three arguments are mutually exclusive. :param duration: The cut's minimal duration after padding. :param num_frames: The cut's total number of frames after padding. :param num_samples: The cut's total number of samples after padding. :param pad_feat_value: A float value that's used for padding the features. By default we assume a log-energy floor of approx. -23 (1e-10 after exp). :param direction: string, 'left', 'right' or 'both'. Determines whether the padding is added before or after the cut. :param preserve_id: When ``True``, preserves the cut ID from before padding. Otherwise, generates a new random ID (default). :param pad_value_dict: Optional dict that specifies what value should be used for padding arrays in custom attributes. :return: a padded MixedCut if duration is greater than this cut's duration, otherwise ``self``. rG)r)rerrrrrr)rIr) rBrerrrrrrrrCrCrDr~s z MixedCut.padraffix_idrecording_fieldcsB|jsJdt|r|jdn|jfdd|jDdS)a Return a new ``MixedCut`` that will lazily resample the audio while reading it. This operation will drop the feature manifest, if attached. It does not affect the supervision. :param sampling_rate: The new sampling rate. :param affix_id: Should we modify the ID (useful if both versions of the same cut are going to be present in a single manifest). :param recording_field: which recording field to resample. :return: a modified copy of the current ``MixedCut``. z-Cannot resample a MixedCut without Recording._rsc$g|]}t||jjddqS))rr)r2r:resamplerrrrCrDraz%MixedCut.resample..rrwrXrYrZ)rBrrrrCrrDrs zMixedCut.resampleopusGz?codeccompression_levelcompress_custom_fieldscs0|jsJdt|jfdd|jDdS)a  Return a copy of this Cut that has Recordings in its sub-Cuts processed by a lossy encoding. :param codec: The codec to use for compression. Supported codecs are "opus", "mp3", "vorbis", "gsm". :param compression_level: The level of compression (from 0.0 to 1.0, higher values correspond to higher compression). :param compress_custom_fields: Whether to also compress any custom recording fields in sub-Cuts. :return: A modified :class:`~lhotse.MixedCut` containing audio processed by a codec z/Cannot compress a MixedCut without a Recording.c s$g|]}t||jdqSr)r2r:compressrrrrrCrDrasz%MixedCut.compress..rr)rBrrrrCr rDr szMixedCut.compressfactorcTjsJdjrtdtrjdnjfddjDdS)a Return a new ``MixedCut`` that will lazily perturb the speed while loading audio. The ``num_samples``, ``start`` and ``duration`` fields of the underlying Cuts (and their Recordings and SupervisionSegments) are updated to reflect the shrinking/extending effect of speed. We are also updating the offsets of all underlying tracks. :param factor: The speed will be adjusted this many times (e.g. factor=1.1 means 1.1x faster). :param affix_id: When true, we will modify the ``MixedCut.id`` field by affixing it with "_sp{factor}". :return: a modified copy of the current ``MixedCut``. z5Cannot perturb speed on a MixedCut without Recording.zAttempting to perturb speed on a MixedCut that references pre-computed features. The feature manifest(s) will be detached, as we do not support feature-domain speed perturbation._spc Fg|]}t||jjdttt|jjdjdddqSr r)rr rirj)r:r=)r2r: perturb_speedrlr6r1r=rrfrr rBrCrDra$z*MixedCut.perturb_speed..rrwruloggingwarningrXrYrZrBr rrCrrDrszMixedCut.perturb_speedcr )a Return a new ``MixedCut`` that will lazily perturb the tempo while loading audio. Compared to speed perturbation, tempo preserves pitch. The ``num_samples``, ``start`` and ``duration`` fields of the underlying Cuts (and their Recordings and SupervisionSegments) are updated to reflect the shrinking/extending effect of tempo. We are also updating the offsets of all underlying tracks. :param factor: The tempo will be adjusted this many times (e.g. factor=1.1 means 1.1x faster). :param affix_id: When true, we will modify the ``MixedCut.id`` field by affixing it with "_tp{factor}". :return: a modified copy of the current ``MixedCut``. z5Cannot perturb tempo on a MixedCut without Recording.zAttempting to perturb tempo on a MixedCut that references pre-computed features. The feature manifest(s) will be detached, as we do not support feature-domain tempo perturbation._tpc rr)r2r: perturb_temporlr6r1r=rrfrrCrDra-rz*MixedCut.perturb_tempo..rrrrCrrDrszMixedCut.perturb_tempocsR|jsJd|jrtdtr|jdn|jfdd|jDdS)a Return a new ``MixedCut`` that will lazily perturb the volume while loading audio. Recordings of the underlying Cuts are updated to reflect volume change. :param factor: The volume will be adjusted this many times (e.g. factor=1.1 means 1.1x louder). :param affix_id: When true, we will modify the ``MixedCut.id`` field by affixing it with "_vp{factor}". :return: a modified copy of the current ``MixedCut``. z6Cannot perturb volume on a MixedCut without Recording.zAttempting to perturb volume on a MixedCut that references pre-computed features. The feature manifest(s) will be detached, as we do not support feature-domain volume perturbation._vpcr)rr)r2r:perturb_volumerfrr rCrDraVrz+MixedCut.perturb_volume..rrrrCrrDr@s  zMixedCut.perturb_volumehardgain_db normalize oversamplingcsX|jsJd|jrtdtr|jdn|jfdd|jDdS)aM Return a new ``MixedCut`` that will lazily apply clipping while loading audio. Recordings of the underlying Cuts are updated to reflect clipping change. :param hard: If True, apply hard clipping (sharp cutoff); otherwise, apply soft clipping (saturation). :param gain_db: The amount of gain in decibels to apply before clipping. :param normalize: If True, normalize the input signal to 0 dBFS before applying clipping. :param oversampling: If provided, we will oversample the input signal by the given integer factor before applying saturation and then downsample back to the original sampling rate. :param affix_id: When true, we will modify the ``MixedCut.id`` field by affixing it with "_cl{gain_db}". :return: a modified copy of the current ``MixedCut``. z8Cannot apply saturation on a MixedCut without Recording.zAttempting to apply saturation on a MixedCut that references pre-computed features. The feature manifest(s) will be detached, as we do not support feature-domain saturation._clc s*g|]}t||jjddqS))rr r!r"rr)r2r:clip_amplituderfrr rr!r"rCrDras z+MixedCut.clip_amplitude..rr)rBrr r!r"rrCr%rDr$_s zMixedCut.clip_amplitudetarget mix_firstr!cs|jsJd|jrtdd|_|r=|jdur|jng}|tdt |r7|j d|dS|j |dSt rH|j dn|j fdd|j Dd S) a Return a new ``MixedCut`` that will lazily apply loudness normalization. :param target: The target loudness in dBFS. :param mix_first: If true, we will mix the underlying cuts before applying loudness normalization. If false, we cannot guarantee that the resulting cut will have the target loudness. :param affix_id: When true, we will modify the ``DataCut.id`` field by affixing it with "_ln{target}". :return: a modified copy of the current ``DataCut``. zDCannot apply loudness normalization on a MixedCut without Recording.zAttempting to normalize loudness on a MixedCut that references pre-computed features. The feature manifest will be detached, as we do not support feature-domain loudness normalization.N)r&_lnrYr[cr))r&rr)r2r:normalize_loudnessrfrr&rCrDrasz/MixedCut.normalize_loudness..r) rwrurrrr[copyrrr2rYrXrZ)rBr&r'rr[rCr+rDr*s4 zMixedCut.normalize_loudnessr rir_recordingrnormalize_output early_only rir_channels room_rng_seedsource_rng_seedc s|jsJd|jrtddus"tfdd|Ds"Jdt|dks5t|t|jks5JdttdurEt |j |gt|j} |dur_fd d |jD} | d }|rdurtd d l m } | |j |d } nd} |jdur|jng} | t|dur|nd g| dt|r|j d| dS|j | dSt|dkr|t|j}tr|j dn|j fdd t|j|| DdS)a Return a new ``MixedCut`` that will convolve the audio with the provided impulse response. If no ``rir_recording`` is provided, we will generate an impulse response using a fast random generator (https://arxiv.org/abs/2208.04101). :param rir_recording: The impulse response to use for convolving. :param normalize_output: When true, output will be normalized to have energy as input. :param early_only: When true, only the early reflections (first 50 ms) will be used. :param affix_id: When true, we will modify the ``MixedCut.id`` field by affixing it with "_rvb". :param rir_channels: The channels of the impulse response to use. By default, first channel is used. If only one channel is specified, all tracks will be convolved with this channel. If a list is provided, it must contain as many channels as there are tracks such that each track will be convolved with one of the specified channels. :param room_rng_seed: Seed for the room configuration. :param source_rng_seed: Seed for the source position. :param mix_first: When true, the mixing will be done first before convolving with the RIR. This effectively means that all tracks will be convolved with the same RIR. If you are simulating multi-speaker mixtures, you should set this to False. :return: a modified copy of the current ``MixedCut``. z;Cannot apply reverberation on a MixedCut without Recording.zAttempting to reverberate a MixedCut that references pre-computed features. The feature manifest(s) will be detached, as we do not support feature-domain reverberation.Nc3s|]}|jkVqdSr?)ro)r^c)r-rCrDrg  z&MixedCut.reverb_rir..z(Invalid channel index in `rir_channels`.rGz`Invalid number of channels in `rir_channels`, must be either 1 or equal to the number of tracks.csg|] }t|jjqSrC)r3r:rYrf) uuid4_strrCrDrasz'MixedCut.reverb_rir..r)FastRandomRIRGenerator)sr room_seed source_seed)rirr.r/r0 rir_generator_rvbr)c s6g|]\}}}t||jj|g|ddqS))r-r.r/rr0r1r2r)r2r: reverb_rir)r^r_rrseed)rr/r.r-r1rCrDra-s r)rwrurrrrrZrUr8r3rYlhotse.augmentation.utilsr6rr[r,rrr2rXzip) rBr-r.r/rr0r1r2r'source_rng_seedsr6r;r[rC)rr/r.r-r1r5rDr=s"       zMixedCut.reverb_rirmixedcCs|jsdS|jdj}|rQtdd|jddDrQ|jdjj}|}|jdkr7t|j |j f|}nt|j |j |j df|}||d|j df<|Sd}d}d d t |jDd\}} t | jj} |j| jkrx| }| |}tt |jjj||j|d } t |jdddd D]!\} } | |kr|dur|}n| j}| j|| j| j| jjd q|r| j}|j d|j dkr|d|j ddf}|j d|j dkrtj||ddddffdd}|j d|j ksJd|S| jS)a Loads the features of the source cuts and mixes them on-the-fly. :param mixed: when True (default), the features are mixed together (as defined in the mixing function for the extractor). This could result in either a 2D or 3D array. For example, if all underlying tracks are single-channel, the output will be a 2D array of shape (num_frames, num_features). If any of the tracks are multi-channel, the output may be a 3D array of shape (num_frames, num_features, num_channels). :return: A numpy ndarray with features and with shape ``(num_frames, num_features)``, or ``(num_tracks, num_frames, num_features)`` Nrcss|] }t|jtVqdSr?rrrCrCrDrgWz)MixedCut.load_features..rGr.cS0g|]\}}t|jts|jdur||jfqSr?rr:r"r>rrCrCrDran  z*MixedCut.load_features..)feature_extractor base_featsrreference_energyrd)featsr>r=r)axiszInconsistent number of frames in a MixedCut: please report this issue at https://github.com/lhotse-speech/lhotse/issues showing the output of print(cut) or str(cut) on whichload_features() was called.)rurZr:rrrndimnponesrrshaperr%rr;rYcompute_energyr$rtr add_to_mixr>r=r mixed_feats concatenate unmixed_feats)rBrB first_cut padding_valfirst_cut_featsrLreference_featsrJ reference_pos reference_cutrHmixerposr_rCrCrDr@st      "zMixedCut.load_featuresrcCs|jsdS|jdj}d}d}ddt|jDd\}}|j|jkr+|}t|}t|jdj|jdjj||jdj d}t|jddddD]\} } | |kr\|dur\|} n| j} |j | | j | j dqM|owt d d |jD}|o{|}|r|r|j n|j} tt|jd } | jd|j} d| kr| krnn | ddd|jf} | | krdkrnn tj| d d| fgd d} | jd|jksJd|jd| jdd|dd|jpgD}|D]}|| |j} q| S|j} | S)a$ Loads the audios of the source cuts and mix them on-the-fly. :param mixed: When True (default), returns a mix of the underlying tracks. This will return a numpy array with shape ``(num_channels, num_samples)``, where ``num_channels`` is determined by the ``num_channels`` property of the MixedCut. Otherwise returns a numpy array with the number of channels equal to the total number of channels across all tracks in the MixedCut. For example, if it contains a MultiCut with 2 channels and a MonoCut with 1 channel, the returned array will have shape ``(3, num_samples)``. :param mono_downmix: If the MixedCut contains > 1 channels (for e.g. when one of its tracks is a MultiCut), this parameter controls whether the returned array will be down-mixed to a single channel. This down-mixing is done by summing the channels together. :return: A numpy ndarray with audio samples and with shape ``(num_channels, num_samples)`` NrcSrEr?rFrrCrCrDrarGz'MixedCut.load_audio..)rrJ base_offsetrGrK)audior>r=css|]}|jdkVqdS)MultiCutN)r;rfrCrCrDrgr4z&MixedCut.load_audio..r)rrreflect)modez7Inconsistent number of samples in a MixedCut. Expected z but the output of mix has zn. Please report this issue at https://github.com/lhotse-speech/lhotse/issues showing the cut below. MixedCut: cSs$g|]}t|tr |nt|qSrC)rrrL)r^tnfmrCrCrDras )rwrZr:rrYrrrrr=rSr>r}mixed_mono_audio mixed_audior1rrQrrOrr[ unmixed_audio)rBrBrrWreference_audiorJr[r\r]r^r_r` tol_samplesnum_samples_diffr[tfnrCrCrDrst       zMixedCut.load_audiocCs(|jr|jj}|jt|j|jdSdS)N)r)rxrtvideo copy_withr1refps)rBvrCrCrDrl szMixedCut.video with_audioc Cs|jsdSt|jdjjddd|jj|jdjd}t|jddddD]\}}|j |jjddd|jdq'|j }|rG|j ||d}|t |fS) NrF)rp)rnr_rGrK)rlr=)rBr)rxrrZr: load_videorlrnr=rrS mixed_videortorch from_numpy) rBrprBrr]r^r_rlr`rCrCrDrqs  zMixedCut.load_videoc Csxddlm}|t|j\}}|jdd}||}}t|D]\}}|j t || ddd||dq#|S)z^ Display the feature matrix as an image. Requires matplotlib to be installed. rNFrBrG)vminvmax) matplotlib.pyplotpyplotsubplotsrrZrminrmrimshowrOflip transpose) rBpltfigaxesrfminfmaxraxrCrCrDplot_tracks_features.s  &zMixedCut.plot_tracks_featuresc Csddlm}|jdd}|jt|jddd\}}tt|j|D]:\}\}}||d}| t d|j t|||j jD]} | |j j } |j|j| j|j| jddd q@q!|S) zi Display plots of the individual tracks' waveforms. Requires matplotlib to be installed. rNFruT)sharexshareygreeng?)coloralpha)rxryrrzrrZrr@squeezeplotrOlinspacerer:r\trimaxvspanr=rdend) rBrr`rrrr_rr supervisionrCrCrDplot_tracks_audio;s     zMixedCut.plot_tracks_audiocC0|js Jd|jdt|dd|jDdS)zKReturn a copy of the current :class:`MixedCut`, detached from ``features``.zCCannot detach features from a MixedCut with no Recording (cut ID = rcSg|] }t||jdqSr )r2r: drop_featuresrrCrCrDraVz*MixedCut.drop_features..rb)rwrYr2rZrArCrCrDrP zMixedCut.drop_featurescCr)zMReturn a copy of the current :class:`.MixedCut`, detached from ``recording``.zCCannot detach recording from a MixedCut with no Features (cut ID = rcSrr )r2r:drop_recordingrrCrCrDra_rz+MixedCut.drop_recording..rb)rurYr2rZrArCrCrDrYrzMixedCut.drop_recordingcCt|dd|jDdS)PReturn a copy of the current :class:`.MixedCut`, detached from ``supervisions``.cSrr )r2r:drop_supervisionsrrCrCrDrafrz.MixedCut.drop_supervisions..rbrrArCrCrDrbzMixedCut.drop_supervisionscCr)rcSrr )r2r:drop_alignmentsrrCrCrDramrz,MixedCut.drop_alignments..rbrrArCrCrDrirzMixedCut.drop_alignmentscCr)zYReturn a copy of the current :class:`MixedCut`, which doesn't contain any in-memory data.cSrr )r2r:drop_in_memory_datarrCrCrDratrz0MixedCut.drop_in_memory_data..rbrrArCrCrDrprzMixedCut.drop_in_memory_data extractorstorage augment_fn mix_eagerlyc s|r:ddlm}jjddd}j|_|jdjdfddjD|dt dr6j d Sdd Sfd dj D}t j|d S) a Compute the features from this cut, store them on disk, and create a new `MonoCut` object with the feature manifest attached. This cut has to be able to load audio. :param extractor: a ``FeatureExtractor`` instance used to compute the features. :param storage: a ``FeaturesWriter`` instance used to store the features. :param augment_fn: an optional callable used for audio augmentation. :param mix_eagerly: when False, extract and store the features for each track separately, and mix them dynamically when loading the features. When True, mix the audio first and store the mixed features, returning a new ``MonoCut`` instance with the same ID. The returned ``MonoCut`` will not have a ``Recording`` attached. :return: a new ``MonoCut`` instance if ``mix_eagerly`` is True, or returns ``self`` with each of the tracks containing the ``Features`` manifests. rG)MonoCutr)rrrr=rrrcsg|] }t|jdqS)r)r2rYrrArCrDrasz7MixedCut.compute_and_store_features..Nr)rYrdrerrr\rrrcs,g|]}t|jjd|j|jdqS))rrrr)r9r:compute_and_store_featuresr=r>rf)rrrrCrDras r) monorextract_from_samples_and_storerrrYrrer\rrrZrX)rBrrrrr features_inforrC)rrrBrrDrws:      z#MixedCut.compute_and_store_features add_empty shrink_okc s tj}|dkr'|s SddtjDdfddtjD}nc|dks6Jdtjdg}jD]N}t|jjdkrK||q;|jjd}|sr|j|j ks`|jj krrt d|jd |jd |jj d |t |t |jt ||j j d gd dq;t |dS)a Fills the whole duration of a cut with a supervision segment. If the cut has one supervision, its start is set to 0 and duration is set to ``cut.duration``. Note: this may either expand a supervision that was shorter than a cut, or shrink a supervision that exceeds the cut. If there are no supervisions, we will add an empty one when ``add_empty==True``, otherwise we won't change anything. If there are two or more supervisions, we will raise an exception. .. note:: For :class:`.MixedCut`, we expect that only one track contains a supervision. That supervision will be expanded to cover the full MixedCut's duration. :param add_empty: should we add an empty supervision with identical time bounds as the cut. :param shrink_ok: should we raise an error if a supervision would be shrank as a result of calling this method. rcSs g|] \}}t|jtr|qSrCrrrCrCrDras z-MixedCut.fill_supervision..c sNg|]#\}}|kr#t|t|jtj|jj|j jddgddn|qS)rD)rYrrdrerrr\r)r2r:r)rYrr=rerfirst_non_padding_idxrBrCrDras&rGz/Cannot expand more than one supervision (found .z!Cannot shrink supervision (start=z, end=z) to cut (start=0, duration=aU) because the argument `shrink_ok` is `False`. Note: this check prevents accidental data loss for speech recognition, as supervision exceeding a cut indicates there might be some spoken content beyond cuts start or end (an ASR model would be trained to predict more text than spoken in the audio). If this is okay, set `shrink_ok` to `True`.)rdrerrrb) rr\rrZr:rrdr=rrerr2)rBrrn_supsrrsuprCrrDfill_supervisionsP        zMixedCut.fill_supervision transform_fncs@t|}|jD]}t|jtrqfdd|jjD|j_q|S)z Modify the SupervisionSegments by `transform_fn` of this MixedCut. :param transform_fn: a function that modifies a supervision as an argument. :return: a modified MixedCut. csg|]}|qSrC)map)r^r`rrCrDras z-MixedCut.map_supervisions..)r2rZrr:r"r\)rBr new_mixed_cutr_rCrrDmap_supervisionss    zMixedCut.map_supervisions delimiter merge_policycustom_merge_fnc sttd|dkd|dur|nfddt|jdddtd kr'|Sd j}d j}t|| |jd }t d dD}t ddD}t ddt d dDrot ddDrot d|jdtddDddD||d dddDddDddDddDfdd|Dfdd|Dd }|} |g| j_| S)a$ Return a copy of the cut that has all of its supervisions merged into a single segment. The new start is the start of the earliest superivion, and the new duration is a minimum spanning duration for all the supervisions. The text fields are concatenated with a whitespace. .. note:: If you're using individual tracks of a mixed cut, note that this transform drops all the supervisions in individual tracks and assigns the merged supervision in the first :class:`.DataCut` found in ``self.tracks``. :param merge_policy: one of "keep_first" or "delimiter". If "keep_first", we keep only the first segment's field value, otherwise all string fields (including IDs) are prefixed with "cat#" and concatenated with a hash symbol "#". This is also applied to ``custom`` fields. Fields with a ``None`` value are omitted. :param custom_merge_fn: a function that will be called to merge custom fields values. We expect ``custom_merge_fn`` to handle all possible custom keys. When not provided, we will treat all custom values as strings. It will be called roughly like: ``custom_merge_fn(custom_key, [s.custom[custom_key] for s in sups])`` # keep_first)r return_firstNcstt|Sr?)rrU)kvs) merge_func_rCrDrBsz-MixedCut.merge_supervisions..cSrr?rK)rrCrCrDrDrrrGrrDrcs.|]}|jdur|jD]}|VqqdSr?)rkeysr^rrrCrCrDrgN$z.MixedCut.merge_supervisions..csrr?) alignmentrrrCrCrDrgQrcss|] \}}t||VqdSr?)r5)r^s1s2rCrCrDrgUrhcss|]}|jduVqdSr?textrrCrCrDrgUr4zYou are merging overlapping supervisions that have text transcripts. The result is likely to be unusable if you are going to train speech recognition models (cut id: rcs|]}|jVqdSr?)rYrrCrCrDrg_csrr?rrrCrCrDrgar cs|] }|jr|jVqdSr?rrrCrCrDrggrCcsrr?)speakerrrCrCrDrghrCcsrr?)languagerrCrCrDrgirCcsrr?)genderrrCrCrDrgjrCcs&i|]fddDqS)c3.|]}|jdur|jvr|jVqdSr?rrrrCrDrgn9MixedCut.merge_supervisions...rCr^) merge_customsupsrrD ks  z/MixedCut.merge_supervisions..cs&i|]ttfddDqS)c3rr?)rrrrCrDrgzrr)rrr)rrrDrvs  ) rYrrdrerrrrrrrr)rr4rr\rrdrr/rrIr}r@warningswarnrYr)joinrrt) rBrrmstartmend mduration custom_keysalignment_keysmsuprrC)rrrrDmerge_supervisionss`    (    & zMixedCut.merge_supervisions predicatecs t|fdd|jDd}|S)a Modify cut to store only supervisions accepted by `predicate` Example: >>> cut = cut.filter_supervisions(lambda s: s.id in supervision_ids) >>> cut = cut.filter_supervisions(lambda s: s.duration < 5.0) >>> cut = cut.filter_supervisions(lambda s: s.text is not None) :param predicate: A callable that accepts `SupervisionSegment` and returns bool :return: a modified MixedCut c g|] }t||jdqSr )r2r:filter_supervisionsrfrrCrDraz0MixedCut.filter_supervisions..rbr)rBrrrCrrDrs zMixedCut.filter_supervisionsrFcCsPd|vr |dd}d|vrdd|dD}t|ddd|dD|dS) Nr;r[cSg|]}t|qSrC)rrLrrCrCrDraz&MixedCut.from_dict..rYcSrrC)r9rLrfrCrCrDrarrZ)rYrZr[)rJrX)rFr[rCrCrDrLs zMixedCut.from_dictpathc(|js|St|jfdd|jDdS)Ncrr )r2r:with_features_path_prefixrrrCrDrarz6MixedCut.with_features_path_prefix..r)rurXrYrZrBrrCrrDr z"MixedCut.with_features_path_prefixcr)Ncrr )r2r:with_recording_path_prefixrrrCrDrarz7MixedCut.with_recording_path_prefix..rrrrCrrDrrz#MixedCut.with_recording_path_prefixcCrsr?)_first_non_padding_trackr:rArCrCrDfirst_non_padding_cutrvzMixedCut.first_non_padding_cutcCsdd|jDdS)NcSrrCrrrCrCrDrarz4MixedCut.first_non_padding_track..rrbrArCrCrDfirst_non_padding_trackr~z MixedCut.first_non_padding_trackcCrr?)rrArCrCrDrtzMixedCut._first_non_padding_cutcCrr?)rrArCrCrDrrz!MixedCut._first_non_padding_track)rTTT)r)FN)rrF)T)Fr<TrT)TF)TTF)rMrX)NT)rN)jr@rQrRrSrUrTr r9r[rrpropertyr)r\r.rdrerintrrboolrurwrxr{rrrrrrrorrWrNr rrr&rrr(rrrrrrOndarrayrr!rrrr rr rrr+floatrrrr rrrr$r*r=r7rrrrlrsTensorrqrrrrrrrr#r'rrrrrr rrrVrLr-rrrrrtr __classcell__rCrCrrDrXesL $     7E   ! "    -   -/! 0 2  h `      = U   n     rX)Srr dataclassesr functoolsrrioroperatorrtypingrrr r r r r rrrnumpyrOrs intervaltreerrrr lhotse.audiorrrlhotse.audio.backendrlhotse.audio.mixerrrrlhotse.augmentationrrrrlhotse.augmentation.compressrlhotse.cut.baser lhotse.cut.datar!lhotse.cut.paddingr"lhotse.featuresr#r$r%lhotse.features.baser&lhotse.features.ior' lhotse.imager(lhotse.supervisionr) lhotse.utilsr*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9rXrCrCrCrDs:   0           D"