o ,iJ @sddlZddlZddlZddlZddlmZmZmZmZm Z m Z m Z m Z m Z mZddlmZddlmZmZddlmZmZgdZe dd d Ze d ZeeefZe ed fZe d eeZGdddeeZGdddeeeeZGdddee ed fZ GdddeeZ!GdddeeZ"GdddeZ#GdddeeZ$efdeede ee%e&fde ede e$efd d!Z'dS)"N) castDictGenericIterableListOptionalSequenceTupleTypeVarUnion) deprecated)default_generatorrandperm) GeneratorTensor)DatasetIterableDataset TensorDataset StackDataset ConcatDataset ChainDatasetSubset random_splitT_coT) covariantT.T_stackc@s(eZdZdZdefddZd dd Zd S) raAn abstract class representing a :class:`Dataset`. All datasets that represent a map from keys to data samples should subclass it. All subclasses should overwrite :meth:`__getitem__`, supporting fetching a data sample for a given key. Subclasses could also optionally overwrite :meth:`__len__`, which is expected to return the size of the dataset by many :class:`~torch.utils.data.Sampler` implementations and the default options of :class:`~torch.utils.data.DataLoader`. Subclasses could also optionally implement :meth:`__getitems__`, for speedup batched samples loading. This method accepts list of indices of samples of batch and returns list of samples. .. note:: :class:`~torch.utils.data.DataLoader` by default constructs an index sampler that yields integral indices. To make it work with a map-style dataset with non-integral indices/keys, a custom sampler must be provided. returncCstd)Nz3Subclasses of Dataset should implement __getitem__.)NotImplementedErrorselfindexr#V/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/utils/data/dataset.py __getitem__>szDataset.__getitem__other Dataset[T_co]ConcatDataset[T_co]cC t||gSN)rr!r&r#r#r$__add__E zDataset.__add__N)r&r'rr()__name__ __module__ __qualname____doc__rr%r,r#r#r#r$r+src@s"eZdZdZdeefddZdS)raHAn iterable Dataset. All datasets that represent an iterable of data samples should subclass it. Such form of datasets is particularly useful when data come from a stream. All subclasses should overwrite :meth:`__iter__`, which would return an iterator of samples in this dataset. When a subclass is used with :class:`~torch.utils.data.DataLoader`, each item in the dataset will be yielded from the :class:`~torch.utils.data.DataLoader` iterator. When :attr:`num_workers > 0`, each worker process will have a different copy of the dataset object, so it is often desired to configure each copy independently to avoid having duplicate data returned from the workers. :func:`~torch.utils.data.get_worker_info`, when called in a worker process, returns information about the worker. It can be used in either the dataset's :meth:`__iter__` method or the :class:`~torch.utils.data.DataLoader` 's :attr:`worker_init_fn` option to modify each copy's behavior. Example 1: splitting workload across all workers in :meth:`__iter__`:: >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_DATALOADER) >>> # xdoctest: +SKIP("Fails on MacOS12") >>> class MyIterableDataset(torch.utils.data.IterableDataset): ... def __init__(self, start, end): ... super(MyIterableDataset).__init__() ... assert end > start, "this example code only works with end >= start" ... self.start = start ... self.end = end ... ... def __iter__(self): ... worker_info = torch.utils.data.get_worker_info() ... if worker_info is None: # single-process data loading, return the full iterator ... iter_start = self.start ... iter_end = self.end ... else: # in a worker process ... # split workload ... per_worker = int(math.ceil((self.end - self.start) / float(worker_info.num_workers))) ... worker_id = worker_info.id ... iter_start = self.start + worker_id * per_worker ... iter_end = min(iter_start + per_worker, self.end) ... return iter(range(iter_start, iter_end)) ... >>> # should give same set of data as range(3, 7), i.e., [3, 4, 5, 6]. >>> ds = MyIterableDataset(start=3, end=7) >>> # Single-process loading >>> print(list(torch.utils.data.DataLoader(ds, num_workers=0))) [tensor([3]), tensor([4]), tensor([5]), tensor([6])] >>> # xdoctest: +REQUIRES(POSIX) >>> # Mult-process loading with two worker processes >>> # Worker 0 fetched [3, 4]. Worker 1 fetched [5, 6]. >>> # xdoctest: +IGNORE_WANT("non deterministic") >>> print(list(torch.utils.data.DataLoader(ds, num_workers=2))) [tensor([3]), tensor([5]), tensor([4]), tensor([6])] >>> # With even more workers >>> # xdoctest: +IGNORE_WANT("non deterministic") >>> print(list(torch.utils.data.DataLoader(ds, num_workers=12))) [tensor([3]), tensor([5]), tensor([4]), tensor([6])] Example 2: splitting workload across all workers using :attr:`worker_init_fn`:: >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_DATALOADER) >>> class MyIterableDataset(torch.utils.data.IterableDataset): ... def __init__(self, start, end): ... super(MyIterableDataset).__init__() ... assert end > start, "this example code only works with end >= start" ... self.start = start ... self.end = end ... ... def __iter__(self): ... return iter(range(self.start, self.end)) ... >>> # should give same set of data as range(3, 7), i.e., [3, 4, 5, 6]. >>> ds = MyIterableDataset(start=3, end=7) >>> # Single-process loading >>> print(list(torch.utils.data.DataLoader(ds, num_workers=0))) [3, 4, 5, 6] >>> >>> # Directly doing multi-process loading yields duplicate data >>> print(list(torch.utils.data.DataLoader(ds, num_workers=2))) [3, 3, 4, 4, 5, 5, 6, 6] >>> # Define a `worker_init_fn` that configures each dataset copy differently >>> def worker_init_fn(worker_id): ... worker_info = torch.utils.data.get_worker_info() ... dataset = worker_info.dataset # the dataset copy in this worker process ... overall_start = dataset.start ... overall_end = dataset.end ... # configure the dataset to only process the split workload ... per_worker = int(math.ceil((overall_end - overall_start) / float(worker_info.num_workers))) ... worker_id = worker_info.id ... dataset.start = overall_start + worker_id * per_worker ... dataset.end = min(dataset.start + per_worker, overall_end) ... >>> # Mult-process loading with the custom `worker_init_fn` >>> # Worker 0 fetched [3, 4]. Worker 1 fetched [5, 6]. >>> print(list(torch.utils.data.DataLoader(ds, num_workers=2, worker_init_fn=worker_init_fn))) [3, 5, 4, 6] >>> # With even more workers >>> print(list(torch.utils.data.DataLoader(ds, num_workers=12, worker_init_fn=worker_init_fn))) [3, 4, 5, 6] r&cCr)r*)rr+r#r#r$r,r-zIterableDataset.__add__N)r.r/r0r1rrr,r#r#r#r$rMslrc@sDeZdZUdZeedfed<deddfddZdd Zd d Z dS) rzDataset wrapping tensors. Each sample will be retrieved by indexing tensors along the first dimension. Args: *tensors (Tensor): tensors that have the same size of the first dimension. .tensorsrNcs(tfddDsJd|_dS)Nc3s(|]}dd|dkVqdS)rN)size.0tensorr2r#r$ s z)TensorDataset.__init__..zSize mismatch between tensors)allr2)r!r2r#r7r$__init__s   zTensorDataset.__init__cstfdd|jDS)Nc3|]}|VqdSr*r#r4r"r#r$r8z,TensorDataset.__getitem__..)tupler2r r#r<r$r%szTensorDataset.__getitem__cCs|jddSNr)r2r3r!r#r#r$__len__szTensorDataset.__len__) r.r/r0r1r r__annotations__r:r%rAr#r#r#r$rs  rc@s^eZdZUdZeeefed<dee dee ddfddZ d d Z d e fd d Z ddZdS)raDataset as a stacking of multiple datasets. This class is useful to assemble different parts of complex input data, given as datasets. Example: >>> # xdoctest: +SKIP >>> images = ImageDataset() >>> texts = TextDataset() >>> tuple_stack = StackDataset(images, texts) >>> tuple_stack[0] == (images[0], texts[0]) >>> dict_stack = StackDataset(image=images, text=texts) >>> dict_stack[0] == {'image': images[0], 'text': texts[0]} Args: *args (Dataset): Datasets for stacking returned as tuple. **kwargs (Dataset): Datasets for stacking returned as dict. datasetsargskwargsrNcs|r#|rtdt|d_tfdd|Drtd|_dS|rFt|}t|d_tfdd|DrAtd|_dStd)NztSupported either ``tuple``- (via ``args``) or``dict``- (via ``kwargs``) like input/output, but both types are given.rc3|] }jt|kVqdSr*_lengthlenr5datasetr@r#r$r8z(StackDataset.__init__..zSize mismatch between datasetsc3rFr*rGrJr@r#r$r8rLz%At least one dataset should be passed) ValueErrorrIrHanyrClistvalues)r!rDrEtmpr#r@r$r:s    zStackDataset.__init__cs<t|jtrfdd|jDStfdd|jDS)Ncsi|] \}}||qSr#r#)r5krKr<r#r$ sz,StackDataset.__getitem__..c3r;r*r#rJr<r#r$r8r=z+StackDataset.__getitem__..) isinstancerCdictitemsr>r r#r<r$r%s zStackDataset.__getitem__indicesc Csjt|jtr[dd|D}|jD]F\}}tt|ddrH||}t|t|kr9tdt|dt|t ||D]\}}|||<q>qt ||D] \}}||||<qMq|Sdd|D} |jD]F}tt|ddr||}t|t|krtdt|dt|t || D] \}} | |qqet || D] \}} | ||qqedd| D} | S)NcSsg|]}iqSr#r#r5_r#r#r$ z-StackDataset.__getitems__.. __getitems__z0Nested dataset's output size mismatch. Expected z, got cSsg|]}gqSr#r#rXr#r#r$rZr[cSsg|]}t|qSr#)r>)r5sampler#r#r$rZ)s) rTrCrUrVcallablegetattrr\rIrMzipappend) r!rW dict_batchrRrKrVdatad_sampleidx list_batcht_sample tuple_batchr#r#r$r\sP      zStackDataset.__getitems__cC|jSr*)rHr@r#r#r$rA,szStackDataset.__len__)r.r/r0r1r r>rUrBrrr:r%rOr\rAr#r#r#r$rs  %rcseZdZUdZeeeed<eeed<e ddZ de eddffdd Z d d Z d d ZeededddZZS)rzDataset as a concatenation of multiple datasets. This class is useful to assemble different existing datasets. Args: datasets (sequence): List of datasets to be concatenated rCcumulative_sizescCs6gd}}|D]}t|}|||||7}q|Sr?)rIra)sequencerselr#r#r$cumsum<s  zConcatDataset.cumsumrNcsZtt||_t|jdksJd|jD] }t|tr#Jdq||j|_dS)Nrz(datasets should not be an empty iterablez.ConcatDataset does not support IterableDataset) superr:rOrCrIrTrrprj)r!rCd __class__r#r$r:Es   zConcatDataset.__init__cCs |jdS)Nrjr@r#r#r$rAO zConcatDataset.__len__cCsf|dkr| t|krtdt||}t|j|}|dkr#|}n ||j|d}|j||S)Nrz8absolute value of index should not exceed dataset length)rIrMbisect bisect_rightrjrC)r!re dataset_idx sample_idxr#r#r$r%Rs zConcatDataset.__getitem__z>`cummulative_sizes` attribute is renamed to `cumulative_sizes`)categorycCrir*rvr@r#r#r$cummulative_sizes`szConcatDataset.cummulative_sizes)r.r/r0r1rrrrBint staticmethodrprr:rAr%propertyr FutureWarningr~ __classcell__r#r#rsr$r0s    rcs>eZdZdZdeeddffdd ZddZd d ZZ S) ra_Dataset for chaining multiple :class:`IterableDataset` s. This class is useful to assemble different existing dataset streams. The chaining operation is done on-the-fly, so concatenating large-scale datasets with this class will be efficient. Args: datasets (iterable of IterableDataset): datasets to be chained together rCrNcst||_dSr*)rqr:rC)r!rCrsr#r$r:ts  zChainDataset.__init__ccs.|jD]}t|tsJd|EdHqdS)N*ChainDataset only supports IterableDataset)rCrTr)r!rrr#r#r$__iter__xs  zChainDataset.__iter__cCs2d}|jD]}t|tsJd|t|7}q|S)Nrr)rCrTrrI)r!totalrrr#r#r$rAs zChainDataset.__len__) r.r/r0r1rrr:rrArr#r#rsr$ris  rc@sreZdZUdZeeed<eeed<deedeeddfddZ dd Z de ede efd d Z d d Z dS)rz Subset of a dataset at specified indices. Args: dataset (Dataset): The whole Dataset indices (sequence): Indices in the whole set selected for subset rKrWrNcCs||_||_dSr*rKrW)r!rKrWr#r#r$r:s zSubset.__init__cs2t|trjfdd|DSjj|S)Ncg|]}j|qSr#rW)r5ir@r#r$rZz&Subset.__getitem__..)rTrOrKrW)r!rer#r@r$r%s zSubset.__getitem__cs>ttjddrjfdd|DSfdd|DS)Nr\crr#rr5rer@r#r$rZrz'Subset.__getitems__..csg|] }jj|qSr#rrr@r#r$rZs)r^r_rKr\)r!rWr#r@r$r\szSubset.__getitems__cCs t|jSr*)rIrWr@r#r#r$rArwzSubset.__len__)r.r/r0r1rrrBrrr:r%rr\rAr#r#r#r$rs    rrKlengths generatorrc s6tt|drnt|dkrng}t|D]$\}}|dks |dkr(td|dttt|}||qtt|}t |D]}|t|}||d7<qE|}t|D]\}} | dkrmt d|dq\t|tkrztdt t||d ttt|}fd d tt||DS) a Randomly split a dataset into non-overlapping new datasets of given lengths. If a list of fractions that sum up to 1 is given, the lengths will be computed automatically as floor(frac * len(dataset)) for each fraction provided. After computing the lengths, if there are any remainders, 1 count will be distributed in round-robin fashion to the lengths until there are no remainders left. Optionally fix the generator for reproducible results, e.g.: Example: >>> # xdoctest: +SKIP >>> generator1 = torch.Generator().manual_seed(42) >>> generator2 = torch.Generator().manual_seed(42) >>> random_split(range(10), [3, 7], generator=generator1) >>> random_split(range(30), [0.3, 0.3, 0.4], generator=generator2) Args: dataset (Dataset): Dataset to be split lengths (sequence): lengths or fractions of splits to be produced generator (Generator): Generator used for the random permutation. rxrzFraction at index z is not between 0 and 1zLength of split at index z- is 0. This might result in an empty dataset.zDSum of input lengths does not equal the length of the input dataset!)rcs&g|]\}}t|||qSr#)r)r5offsetlengthrr#r$rZsz random_split..)mathisclosesum enumeraterMrfloorrIrarangewarningswarnrtolistrrr` itertools accumulate) rKrrsubset_lengthsrfracn_items_in_split remainder idx_to_add_atrr#rr$rs:     r)(ryrrrtypingrrrrrrrr r r typing_extensionsr torchr rrr__all__rrstrT_dictT_tuplerrrrrrrrrfloatrr#r#r#r$s>0     "tW9 $