o zi$@sddlZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z ddl m Z ddlmZddlZddlmZddlmZmZmZddlmZddlmZdd lmZd ZGd d d eZd ededefddZGdddeZdS)N)AnyCallableOptionalSizedTupleUnion) HTTPError)warn)Tensor) DataLoaderDataset random_split) _IS_WINDOWS)LightningDataModule)_TORCHVISION_AVAILABLEz./datac seZdZdZdZdZdZdZ d,ded e d e d e d e d df fdd Z de d eee ffddZd e fddZed efddZded e fddZd-d e d dfddZded dfddZed.d!ed"e d#ed eeeffd$d%Zed/d'ed(ee efd)ee efd efd*d+ZZS)0_MNISTa<Carbon copy of ``tests_pytorch.helpers.datasets.MNIST``. We cannot import the tests as they are not distributed with the package. See https://github.com/Lightning-AI/lightning/pull/7614#discussion_r671183652 for more context. .. warning:: This is meant for testing/debugging and is experimental. )zChttps://pl-public-data.s3.amazonaws.com/MNIST/processed/training.ptz?https://pl-public-data.s3.amazonaws.com/MNIST/processed/test.ptz training.ptztest.ptcompleteTg_)Ǻ?gGr?roottrain normalizedownloadkwargsreturnNc sZt||_||_||_|||jr|jn|j}|t j |j |\|_ |_dSN)super__init__rrr prepare_dataTRAIN_FILE_NAMETEST_FILE_NAME _try_loadospathjoincached_folder_pathdatatargets)selfrrrrr data_file __class__\/home/ubuntu/.local/lib/python3.10/site-packages/pytorch_lightning/demos/mnist_datamodule.pyr5s  "z_MNIST.__init__idxcCsV|j|d}t|j|}|jdur't|jdkr'|j|g|jR}||fS)Nr)r%float unsqueezeintr&rlennormalize_tensor)r'r-imgtargetr+r+r, __getitem__Bs z_MNIST.__getitem__cCs t|jSr)r2r%r'r+r+r,__len__Ks z_MNIST.__len__cCstj|jd|jS)NMNIST)r!r"r#rcache_folder_namer7r+r+r,r$Nsz_MNIST.cached_folder_path data_foldercCs4d}|j|jfD]}|otjtj||}q|S)NT)rrr!r"isfiler#)r'r;existingfnamer+r+r, _check_existsRsz_MNIST._check_existscCs4|r||js||j||jstddS)NzDataset not found.)r?r$ _download RuntimeError)r'rr+r+r,rXs   z_MNIST.prepare_datacCsRtj|dd|jD]}td|tj|tj|}tj ||q dS)NT)exist_okz Downloading ) r!makedirs RESOURCESlogginginfor"r#basenameurllibrequest urlretrieve)r'r;urlfpathr+r+r,r@^s  z_MNIST._download? path_datatrialsdeltac Csd\}}|s Jdtj|sJd|t|D]'}zt|}WntyA}z|}t|t WYd}~qd}~ww|dusIJ|durO||S)zHResolving loading from the same time from multiple concurrent processes.)NNz!at least some trial has to be setzmissing file: N) r!r"r<rangetorchload Exceptiontimesleeprandom)rOrPrQres exception_exr+r+r,r es    z_MNIST._try_loadtensormeanstdcCs8tj||j|jd}tj||j|jd}|||S)N)dtypedevice)rS as_tensorrarbsubdiv)r^r_r`r+r+r,r3zsz_MNIST.normalize_tensor)TrT)T)rMrN)r]rN)__name__ __module__ __qualname____doc__rDrrr:strbooltuplerrr1rr r6r8propertyr$r?rr@ staticmethodr/r rr3 __classcell__r+r+r)r,r"s>   &6rargsrrc Osttdd }|r6zddlm}|tddWnty5}ztd|dd}WYd}~nd}~ww|s>td t}||i|S) NPL_USE_MOCKED_MNISTFr)r9T)rzError z) downloading `torchvision.datasets.MNIST`zD`torchvision.datasets.MNIST` not available. Using our hosted version) rkr!getenvtorchvision.datasetsr9_DATASETS_PATHrprintr)rprtorchvision_mnist_availabler9r\r+r+r,r9s r9cseZdZdZdZedddddfded ed ed ed ed eddffdd Z e defddZ d!ddZ deddfddZ defddZdefddZdefddZe deefdd ZZS)"MNISTDataModulezStandard MNIST, train, val, test splits and transforms. >>> MNISTDataModule() # doctest: +ELLIPSIS <...mnist_datamodule.MNISTDataModule object at ...> mnistiF* data_dir val_split num_workersrseed batch_sizerNcsNt|rtrtd|dd}||_||_||_||_||_||_ dS)an Args: data_dir: where to save/load the data val_split: how many of the training images to use for the validation split num_workers: how many workers to use for loading data normalize: If true applies image normalize seed: starting seed for RNG. batch_size: desired batch size. zYou have requested num_workers=zA on Windows, but currently recommended is 0, so we set it for yourN) rrrr r|r}r~rrr)r'r|r}r~rrrr)r+r,rs   zMNISTDataModule.__init__cCsdS)N r+r7r+r+r, num_classesszMNISTDataModule.num_classescCs$t|jdddt|jddddS)zSaves MNIST files to `data_dir`TrrFN)r9r|r7r+r+r,rszMNISTDataModule.prepare_datastagecCsr|jrd|jini}t|jfddd|}t|tsJt|}t|||j|jgt dd\|_ |_ dS)z"Split the train and valid dataset. transformTFrrz) generatorN) default_transformsr9r| isinstancerr2r r}rS Generator manual_seed dataset_train dataset_val)r'rextradataset train_lengthr+r+r,setupszMNISTDataModule.setupcCst|j|jd|jdddS)z7MNIST train set removes a subset to use for validation.Trshuffler~ drop_last pin_memory)r rrr~r7r+r+r,train_dataloaderz MNISTDataModule.train_dataloadercCst|j|jd|jdddS)z?MNIST val set uses a subset of the training set for validation.FTr)r rrr~r7r+r+r,val_dataloaderrzMNISTDataModule.val_dataloadercCsD|jrd|jini}t|jfddd|}t||jd|jdddS)z#MNIST test set uses the test split.rFrTr)rr9r|r rr~)r'rrr+r+r,test_dataloaderszMNISTDataModule.test_dataloadercCsFtsdSddlm}|jr|||jdddg}|S|}|S)Nr) transforms)g?)r_r`)r torchvisionrrComposeToTensor Normalize)r'rmnist_transformsr+r+r,rs  z"MNISTDataModule.default_transforms)rN)rfrgrhrinamertrjr1rkrrmrrrr rrrrrrror+r+r)r,rwsB"     rw) rEr!rXrVrHtypingrrrrrr urllib.errorrwarningsr rSr torch.utils.datar r r "lightning_fabric.utilities.importsrpytorch_lightningr#pytorch_lightning.utilities.importsrrtrr9rwr+r+r+r,s$        _