o „o™i ã@s¬dZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZddlmZddlZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$dd l%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7m8Z8ddl9m:Z:m;Z;ddlm?Z?m@Z@ddlAmBZBmCZCddlDmEZEddlFmGZGmHZHmIZImJZJddlKmLZLddlMmNZNddlOmPZPddlQmRZRdd lSmTZTmUZUmVZVdd!lWmXZXdd"lYmZZZm[Z[m\Z\m]Z]dd#l^m_Z_m`Z`dd$lambZbdd%lcmdZdmeZedd&lfmgZgdd'lhmiZidd(ljmkZkmlZldd)lmmnZndd*lompZpmqZqmrZrmsZsmtZtdd+lumvZve wex¡ZyGd,d-„d-ƒZzdS).z!Trainer to automate the training.éN)Úcontextmanager)Ú timedelta)ÚAnyÚDictÚ GeneratorÚIterableÚListÚOptionalÚUnion)Úproxy)Ú Optimizer)Úconvert_tensors_to_scalars)Ú_is_local_file_protocol)Ú_PATH)Ú Accelerator)ÚCallbackÚ CheckpointÚ EarlyStoppingÚ ProgressBar)ÚLightningDataModule)ÚLogger)Ú CSVLogger)ÚTensorBoardLogger)Ú_log_hyperparams)Ú_PredictionLoopÚ_TrainingEpochLoop)Ú_EvaluationLoop)Ú_FitLoop)Ú_parse_loop_limitsÚ_reset_progress)Ú _PLUGIN_INPUTÚ Precision)ÚProfiler)ÚParallelStrategyÚStrategy)ÚcallÚsetup)Ú_verify_loop_configurations)Ú _LITERAL_WARNÚ_PRECISION_INPUTÚ_PRECISION_INPUT_STRÚ_AcceleratorConnector)Ú_CallbackConnector)Ú_CheckpointConnector)Ú_DataConnector)Ú_LoggerConnector)Ú _OUT_DICTÚ _PBAR_DICTÚ_ResultCollection)Ú_SignalConnector)Ú RunningStageÚ TrainerFnÚ TrainerStateÚ TrainerStatus)ÚGradClipAlgorithmTypeÚparsing)Ú_defaults_from_env_vars)Ú_maybe_unwrap_optimizedÚ!_verify_strategy_supports_compile)ÚMisconfigurationException)Ú is_overridden)Úrank_zero_infoÚrank_zero_warn)Ú isolate_rng)Ú_EVALUATE_OUTPUTÚ_PREDICT_OUTPUTÚEVAL_DATALOADERSÚTRAIN_DATALOADERSÚLRSchedulerConfig)ÚPossibleUserWarningcRs eZdZedddddddddddddddddddddddddddddddddddddddd œ'd eeefd eeefd eee ee fd e de e de ee e e efde eeeefdee efde e de e de de e de eeeeee ffde ee efde ee efde ee efde ee efdee efde ee efde e de e de e d e ed!e ed"e ed#e d$e ee efd%e ed&e eeefd'e ed(ed)ed*e eeefd+ed,ed-e eeeefd.ed/e d0e ed1dfP‡fd2d3„ƒZ    dçd4d5d6e eeefd7e ed8e ed9e ed1df d:d;„Z    dçd4d5d6e eeefd7e ed8e ed9e ed1df de eeefd9e ed?ed8e ed1ef d@dA„Z     dèd4e d5d>e eeefd9e ed?ed8e ed1e eeeff dBdC„Z     dèd4e d5d>e eeefd9e ed?ed8e ed1ef dDdE„Z!     dèd4e d5d>e eeefd9e ed?ed8e ed1e eeeff dFdG„Z"     déd4e d5d>e eeefd8e edHe ed9e ed1e ef dIdJ„Z#     déd4e d5d>e eeefd8e edHe ed9e ed1e ef dKdL„Z$ dêd4d5d9e ed1e eeeffdMdN„Z%dëdOdP„Z&d1e eeeffdQdR„Z'dëdSdT„Z(dëdUdV„Z)e*dêdWe ed1e+fdXdY„ƒZ,dZe-d[e-d1dfd\d]„Z. e/d1efd^d_„ƒZ0e/d1efd`da„ƒZ1e/d1e2fdbdc„ƒZ3e/d1e fddde„ƒZ4e/d1e fdfdg„ƒZ5e/d1e fdhdi„ƒZ6e/d1e fdjdk„ƒZ7e/d1e fdldm„ƒZ8e/d1ee fdndo„ƒZ9e/d1e fdpdq„ƒZ:e/dìdrds„ƒZ;e/d1ee<fdtdu„ƒZ=e=j>dvee<d1dfdwdu„ƒZ=e/d1ee?fdxdy„ƒZ@e/d1eAfdzd{„ƒZBe/d1e e-fd|d}„ƒZCe/d1e eDjEjFfd~d„ƒZG e/d1e efd€d„ƒZHe/d1efd‚dƒ„ƒZIe/d1e eee-ffd„d…„ƒZJe/d1efd†d‡„ƒZKe/d1efdˆd‰„ƒZLe/d1e eMfdŠd‹„ƒZNe/d1eeMfdŒd„ƒZOe/d1e ePfdŽd„ƒZQe/d1eePfdd‘„ƒZRe/d1e eSfd’d“„ƒZTe/d1e efd”d•„ƒZUeUj>d9e ed1dfd–d•„ƒZU díd—ed˜ed™e e-d1dfdšd›„ZV e/d1efdœd„ƒZWe/d1efdždŸ„ƒZXeXj>d ed1dfd¡dŸ„ƒZXe/d1efd¢d£„ƒZYeYj>d ed1dfd¤d£„ƒZYe/d1efd¥d¦„ƒZZeZj>d ed1dfd§d¦„ƒZZe/d1efd¨d©„ƒZ[e[j>d ed1dfdªd©„ƒZ[e/d1efd«d¬„ƒZ\e/d1efd­d®„ƒZ]e]j>d ed1dfd¯d®„ƒZ]e/d1efd°d±„ƒZ^ e/d1e fd²d³„ƒZ_e/d1e fd´dµ„ƒZ`e/d1e e fd¶d·„ƒZae/d1e e fd¸d¹„ƒZbe/d1e fdºd»„ƒZce/d1e e fd¼d½„ƒZde/d1efd¾d¿„ƒZee/d1e efdÀdÁ„ƒZfe/d1e efdÂdăZge/d1e efdÄdÅ„ƒZhe/d1e efdÆdÇ„ƒZie/d1ee effdÈdÉ„ƒZje/d1eee effdÊdË„ƒZke/d1eee effdÌdÍ„ƒZle/d1eee effdÎdÏ„ƒZme/d1eee effdÐdÑ„ƒZne/d1eofdÒdÓ„ƒZpe/d1e eeqeoerffdÔdÕ„ƒZs e/d1e e fdÖdׄƒZtetj>de e d1dfdØdׄƒZte/d1ee fdÙdÚ„ƒZueuj>dÛe ee d1dfdÜdÚ„ƒZue/d1evfdÝdÞ„ƒZwe/d1evfdßdà„ƒZxe/d1eyfdád℃Zze/d1e e{fdãd䄃Z| e/d1ee effdåd愃Z}‡Z~S)îÚTrainerÚautoéNFéÿÿÿÿgTr)'Ú acceleratorÚstrategyÚdevicesÚ num_nodesÚ precisionÚloggerÚ callbacksÚ fast_dev_runÚ max_epochsÚ min_epochsÚ max_stepsÚ min_stepsÚmax_timeÚlimit_train_batchesÚlimit_val_batchesÚlimit_test_batchesÚlimit_predict_batchesÚoverfit_batchesÚval_check_intervalÚcheck_val_every_n_epochÚnum_sanity_val_stepsÚlog_every_n_stepsÚenable_checkpointingÚenable_progress_barÚenable_model_summaryÚaccumulate_grad_batchesÚgradient_clip_valÚgradient_clip_algorithmÚ deterministicÚ benchmarkÚinference_modeÚuse_distributed_samplerÚprofilerÚdetect_anomalyÚ barebonesÚpluginsÚsync_batchnormÚ!reload_dataloaders_every_n_epochsÚdefault_root_dirrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrÚreturnc') sŠtƒ ¡t |jj›dtƒ›¡|'durt |'¡}'|#|_ |#r±|r*t d|›dƒ‚d}|dur<|dur0)`, `LightningModule.log(...)`, `LightningModule.log_dict(...)`z5 - Sanity checking: `Trainer(num_sanity_val_steps>0)`z0 - Development run: `Trainer(fast_dev_run=True)`z4 - Anomaly detection: `Trainer(detect_anomaly=True)`z% - Profiling: `Trainer(profiler=...)`zÐYou are running in `Trainer(barebones=True)` mode. All features that may impact raw speed have been disabled to facilitate analyzing the Trainer overhead. Specifically, the following features are deactivated:Té2é) rNrLrMrOrprirkrhrPro)rUrT)rWrV)rjz5`gradient_clip_val` should be an int or a float. Got Ú.z`gradient_clip_algorithm` z! is invalid. Allowed algorithms: zYou have turned on `Trainer(detect_anomaly=True)`. This will significantly slow down compute speed and is recommended only for model debugging.)<ÚsuperÚ__init__ÚlogÚdebugÚ __class__Ú__name__ÚlocalsÚosÚfspathrnÚ ValueErrorr?ÚlinesepÚjoinr.Ú_data_connectorr+Ú_accelerator_connectorr/Ú_logger_connectorr,Ú_callback_connectorr-Ú_checkpoint_connectorr3Ú_signal_connectorrÚfit_looprÚ epoch_looprr5Ú VALIDATINGr4Ú validate_loopÚTESTINGÚ test_looprÚ predict_loopreÚon_trainer_initÚ isinstanceÚintÚfloatÚ TypeErrorr8Úsupported_typeÚlowerr=Úsupported_typesrfrgÚ_detect_anomalyr&Ú_log_device_infoÚ should_stopr6ÚstateÚ_init_profilerÚ_init_debugging_flags))ÚselfrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrÚ deactivated©r{©úU/home/ubuntu/.local/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.pyrxWs, V  ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ þ þÿ ö    ÿú ýÿÿÿÿÿ  ÷zTrainer.__init__Úmodelúpl.LightningModuleÚtrain_dataloadersÚval_dataloadersÚ datamoduleÚ ckpt_pathc CsRt|ƒ}||j_t||jƒtj|j_tj |j_ d|_ t   ||j|||||¡dS)anRuns the full optimization routine. Args: model: Model to fit. train_dataloaders: An iterable or collection of iterables specifying training samples. Alternatively, a :class:`~lightning.pytorch.core.datamodule.LightningDataModule` that defines the :class:`~lightning.pytorch.core.hooks.DataHooks.train_dataloader` hook. val_dataloaders: An iterable or collection of iterables specifying validation samples. datamodule: A :class:`~lightning.pytorch.core.datamodule.LightningDataModule` that defines the :class:`~lightning.pytorch.core.hooks.DataHooks.train_dataloader` hook. ckpt_path: Path/URL of the checkpoint from which training is resumed. Could also be one of two special keywords ``"last"`` and ``"hpc"``. If there is no checkpoint file at the path, an exception is raised. Raises: TypeError: If ``model`` is not :class:`~lightning.pytorch.core.LightningModule` for torch version less than 2.0.0 and if ``model`` is not :class:`~lightning.pytorch.core.LightningModule` or :class:`torch._dynamo.OptimizedModule` for torch versions greater than or equal to 2.0.0 . For more information about multiple dataloaders, see this :ref:`section `. TN)r;rMÚ_lightning_moduler<r5ÚFITTINGr›Úfnr7ÚRUNNINGÚstatusÚtrainingr%Ú_call_and_handle_interruptÚ _fit_impl©ržr£r¥r¦r§r¨r¡r¡r¢Úfitòs"   ÿz Trainer.fitcCs¬t |jj›d¡t|tƒr|}d}|dus|dur#|dur#tdƒ‚|jj||||d|j j dus5J‚|j j |j j |d|j dud}|j||d|j jsQJ‚d|_dS)Nz: trainer fit stagezXYou cannot pass `train_dataloader` or `val_dataloaders` to `trainer.fit(datamodule=...)`)r¥r¦r§T©Úmodel_providedÚmodel_connected©r¨F)ryrzr{r|r‘rr=rƒÚ attach_datar›r«r‡Ú_select_ckpt_pathÚlightning_moduleÚ_runÚstoppedr®r±r¡r¡r¢r°s, ÿÿü zTrainer._fit_implÚ dataloadersÚverbosec Cól|dur|jdur tdƒ‚nt|ƒ}||j_t|j|jƒtj|j_ t j |j_ d|_ t ||j|||||¡S)a•Perform one evaluation epoch over the validation set. Args: model: The model to validate. dataloaders: An iterable or collection of iterables specifying validation samples. Alternatively, a :class:`~lightning.pytorch.core.datamodule.LightningDataModule` that defines the :class:`~lightning.pytorch.core.hooks.DataHooks.val_dataloader` hook. ckpt_path: Either ``"best"``, ``"last"``, ``"hpc"`` or path to the checkpoint you wish to validate. If ``None`` and the model instance was passed, use the current weights. Otherwise, the best model checkpoint from the previous ``trainer.fit`` call will be loaded if a checkpoint callback is configured. verbose: If True, prints the validation results. datamodule: A :class:`~lightning.pytorch.core.datamodule.LightningDataModule` that defines the :class:`~lightning.pytorch.core.hooks.DataHooks.val_dataloader` hook. For more information about multiple dataloaders, see this :ref:`section `. Returns: List of dictionaries with metrics logged during the validation phase, e.g., in model- or callback hooks like :meth:`~lightning.pytorch.LightningModule.validation_step` etc. The length of the list corresponds to the number of validation dataloaders used. Raises: TypeError: If no ``model`` is passed and there was no ``LightningModule`` passed in the previous run. If ``model`` passed is not `LightningModule` or `torch._dynamo.OptimizedModule`. MisconfigurationException: If both ``dataloaders`` and ``datamodule`` are passed. Pass only one of these. RuntimeError: If a compiled ``model`` is passed and the strategy is not supported. Nz^`Trainer.validate()` requires a `LightningModule` when it hasn't been passed in a previous runT)r¹r”r;rMr©r<r5r‹r›r«r7r¬r­Ú validatingr%r¯Ú_validate_impl©ržr£r¼r¨r½r§r¡r¡r¢ÚvalidateDs. ÿÿ  ÿzTrainer.validatecCóÆt |jj›d¡t|tƒr|}d}|dur|rtdƒ‚|dur'|j}d}nd}||j_ |j j |||d|j j dus>J‚|jj|j j |||jdud}|j||d}t|ƒ}|j js^J‚d|_|S)Nz: trainer validate stagezHYou cannot pass both `trainer.validate(dataloaders=..., datamodule=...)`FT)r¦r§r³r¶)ryrzr{r|r‘rr=r¹rŒr½rƒr·r›r«r‡r¸rºr r»r¿©ržr£r¼r¨r½r§r´Úresultsr¡r¡r¢rÀƒó*  ÿ zTrainer._validate_implc Cr¾)aÊPerform one evaluation epoch over the test set. It's separated from fit to make sure you never run on your test set until you want to. Args: model: The model to test. dataloaders: An iterable or collection of iterables specifying test samples. Alternatively, a :class:`~lightning.pytorch.core.datamodule.LightningDataModule` that defines the :class:`~lightning.pytorch.core.hooks.DataHooks.test_dataloader` hook. ckpt_path: Either ``"best"``, ``"last"``, ``"hpc"`` or path to the checkpoint you wish to test. If ``None`` and the model instance was passed, use the current weights. Otherwise, the best model checkpoint from the previous ``trainer.fit`` call will be loaded if a checkpoint callback is configured. verbose: If True, prints the test results. datamodule: A :class:`~lightning.pytorch.core.datamodule.LightningDataModule` that defines the :class:`~lightning.pytorch.core.hooks.DataHooks.test_dataloader` hook. For more information about multiple dataloaders, see this :ref:`section `. Returns: List of dictionaries with metrics logged during the test phase, e.g., in model- or callback hooks like :meth:`~lightning.pytorch.LightningModule.test_step` etc. The length of the list corresponds to the number of test dataloaders used. Raises: TypeError: If no ``model`` is passed and there was no ``LightningModule`` passed in the previous run. If ``model`` passed is not `LightningModule` or `torch._dynamo.OptimizedModule`. MisconfigurationException: If both ``dataloaders`` and ``datamodule`` are passed. Pass only one of these. RuntimeError: If a compiled ``model`` is passed and the strategy is not supported. NzZ`Trainer.test()` requires a `LightningModule` when it hasn't been passed in a previous runT)r¹r”r;rMr©r<r5rr›r«r7r¬r­Útestingr%r¯Ú _test_implrÁr¡r¡r¢Útest°s/ ÿÿ  ÿz Trainer.testcCrÃ)Nz: trainer test stagezDYou cannot pass both `trainer.test(dataloaders=..., datamodule=...)`FT)Útest_dataloadersr§r³r¶)ryrzr{r|r‘rr=r¹rŽr½rƒr·r›r«r‡r¸rºr r»rÇrÄr¡r¡r¢rÈðrÆzTrainer._test_implÚreturn_predictionsc Cr¾)alRun inference on your data. This will call the model forward function to compute predictions. Useful to perform distributed and batched predictions. Logging is disabled in the predict hooks. Args: model: The model to predict with. dataloaders: An iterable or collection of iterables specifying predict samples. Alternatively, a :class:`~lightning.pytorch.core.datamodule.LightningDataModule` that defines the :class:`~lightning.pytorch.core.hooks.DataHooks.predict_dataloader` hook. datamodule: A :class:`~lightning.pytorch.core.datamodule.LightningDataModule` that defines the :class:`~lightning.pytorch.core.hooks.DataHooks.predict_dataloader` hook. return_predictions: Whether to return predictions. ``True`` by default except when an accelerator that spawns processes is used (not supported). ckpt_path: Either ``"best"``, ``"last"``, ``"hpc"`` or path to the checkpoint you wish to predict. If ``None`` and the model instance was passed, use the current weights. Otherwise, the best model checkpoint from the previous ``trainer.fit`` call will be loaded if a checkpoint callback is configured. For more information about multiple dataloaders, see this :ref:`section `. Returns: Returns a list of dictionaries, one for each provided dataloader containing their respective predictions. Raises: TypeError: If no ``model`` is passed and there was no ``LightningModule`` passed in the previous run. If ``model`` passed is not `LightningModule` or `torch._dynamo.OptimizedModule`. MisconfigurationException: If both ``dataloaders`` and ``datamodule`` are passed. Pass only one of these. RuntimeError: If a compiled ``model`` is passed and the strategy is not supported. See :ref:`Lightning inference section` for more. Nz]`Trainer.predict()` requires a `LightningModule` when it hasn't been passed in a previous runT)r¹r”r;rMr©r<r5Ú PREDICTINGr›r«r7r¬r­Ú predictingr%r¯Ú _predict_impl)ržr£r¼r§rËr¨r¡r¡r¢Úpredicts0 ÿÿ  ÿzTrainer.predictcCs¾t |jj›d¡||j_t|tƒr|}d}|dur!|r!tdƒ‚|dur+|j }d}nd}|j j |||d|j j dus>J‚|jj|j j |||j dud}|j||d}|j jsZJ‚d|_|S)Nz: trainer predict stagezGYou cannot pass both `trainer.predict(dataloaders=..., datamodule=...)`FT)Úpredict_dataloadersr§r³r¶)ryrzr{r|rrËr‘rr=r¹rƒr·r›r«r‡r¸rºr»rÍ)ržr£r¼r§rËr¨r´rÅr¡r¡r¢rÎ^s(   ÿ zTrainer._predict_implcCsJ|jjtjkrt|j|j|j|j|ƒ\}}||j _||j _|j r$t dƒt |dƒr/t  |j¡|j |¡|j ¡|j ¡t|ƒt |jj›d¡|j ¡| ¡t |jj›d¡|j ¡t |¡t |jj›d¡t  |¡|jj!st |jj›d|›¡|j" #|¡|j$ %¡|j$ &¡|j '|¡|jjtjkr²t (|d¡t )|d¡t*|ƒ|jj!rÌt |jj›d|›¡|j" #|¡t |jj›d¡|j" +¡|j" ,¡|j- .¡| /¡}t |jj›d ¡| 0¡|jjtjkr t (|d ¡t )|d ¡t |jj›d ¡t 1|¡t2j3|j_4d|j_5|S) Nz‡`Trainer(barebones=True)` started running. The progress bar is disabled so you might want to manually print the progress in your model.Úhparamsz!: setting up strategy environmentz: preparing dataz: configuring modelz7: restoring module and callbacks from checkpoint path: Ú on_fit_startz: restoring training statez: trainer tearing downÚ on_fit_endz: calling teardown hooks)6r›r«r5rªrrWrVrUrTr‰rnr?Úhasattrr9Úclean_namespacerÑrMÚconnectr†Ú_attach_model_callbacksÚ_attach_model_logging_functionsr'ryrzr{r|Úsetup_environmentÚ_Trainer__setup_profilerrƒÚ prepare_datar%Ú_call_setup_hookÚ_call_configure_modelÚrestore_checkpoint_after_setupr‡Ú_restore_modules_and_callbacksr…Ú reset_resultsÚ reset_metricsr&Ú_call_callback_hooksÚ_call_lightning_module_hookrÚrestore_training_stateÚ resume_endrˆÚregister_signal_handlersÚ _run_stageÚ _teardownÚ_call_teardown_hookr7ÚFINISHEDr­Ústage)ržr£r¨rUrTrÅr¡r¡r¢rºˆshÿÿ                       z Trainer._runcCs8|j ¡|j}|dur| ¡|j ¡|j ¡dS)z¥This is the Trainer's internal teardown, unrelated to the `teardown` hooks in LightningModule and Callback; those are handled by :meth:`_call_teardown_hook`.N)rMÚteardownÚ _active_loopr…rˆ)ržÚloopr¡r¡r¢rèés  zTrainer._teardowncCs¸|j d¡|j ¡|jr|j ¡S|jr|j ¡S|j rTt ƒ |  ¡Wdƒn1s0wYt j  |j¡|j ¡WdƒdS1sMwYdStd|j›ƒ‚)Nz run-stagezUnexpected state )rMÚbarrierr¹Ú zero_gradÚ evaluatingÚ_evaluation_loopÚrunrÍrr®rAÚ_run_sanity_checkÚtorchÚautogradÚset_detect_anomalyr˜r‰Ú RuntimeErrorr›©ržr¡r¡r¢rçôs"     ÿ  ÿþzTrainer._run_stagecCs”|jjj}|jo|jdko|j }|rH|jj}d|_|j   ¡|j   ¡t   |d¡| ¡t   |d¡|j   ¡|j   ¡t|ƒ||j_dSdS)NrTÚon_sanity_check_startÚon_sanity_check_end)r‰rŠÚval_loopÚenable_validationr`Ú restartingr›rëÚsanity_checkingr…ràrár%rârór)ržrüÚshould_sanity_checkrër¡r¡r¢rôs& ÿü        èzTrainer._run_sanity_checkcCsN|jjdusJ‚|jdkr|jnd}t|jƒ|j_|jj|jj||j ddS)NrJ)rëÚ local_rankÚlog_dir) r›r«Ú world_sizerr r¹rlr©r&r)ržrr¡r¡r¢Ú__setup_profiler+szTrainer.__setup_profilerÚ empty_initccshtd|jtdrtdt|jƒj›dtd|jj|d dVWdƒdS1s-wYdS)a•Tensors that you instantiate under this context manager will be created on the device right away and have the right data type depending on the precision setting in the Trainer. The parameters and tensors get created on the device and with the right data type right away without wasting memory being allocated unnecessarily. Args: empty_init: Whether to initialize the model with empty weights (uninitialized memory). If ``None``, the strategy will decide. Some strategies may not support all options. Set this to ``True`` if you are loading a checkpoint into a large model. Úmodel_sharded_context)ÚparentzX`trainer.init_module` cannot fully support proper instantiation of your model with the `zb` strategy. Please instantiate your model inside the`LightningModule.configure_model` hook instead)Úcategory)rN)r>rMr$r@Útyper|rGÚtensor_init_context)ržrr¡r¡r¢Ú init_module1s€ ÿú"ÿzTrainer.init_moduleÚargsÚkwargscOs |jdkrt|i|¤ŽdSdS)zùPrint something only on the first process. If running on multiple machines, it will print from the first process in each machine. Arguments passed to this method are forwarded to the Python built-in :func:`print` function. rN)rÚprint)ržr r r¡r¡r¢rMs ÿz Trainer.printcCs|jjsJ‚|jjS©N)rMrLrùr¡r¡r¢rL[s zTrainer.acceleratorcCó|jjSr)r„rMrùr¡r¡r¢rM`ózTrainer.strategycCrr)rMÚprecision_pluginrùr¡r¡r¢rdrzTrainer.precision_plugincCrr)rMÚ global_rankrùr¡r¡r¢rhrzTrainer.global_rankcCót|jddƒS)Nrr©ÚgetattrrMrùr¡r¡r¢rlózTrainer.local_rankcCr)NÚ node_rankrrrùr¡r¡r¢rqrzTrainer.node_rankcCr)NrrJrrùr¡r¡r¢rvrzTrainer.world_sizecCr)NrOrJrrùr¡r¡r¢rO{ózTrainer.num_nodescCsvt|jtƒr |jjn|jjg}|dusJ‚g}t|ƒD]\}}t|tjƒr.| |j p+|¡qt|t ƒr8| |¡q|S)z List of device indexes per node.N) r‘rMr#Úparallel_devicesÚ root_deviceÚ enumeraterõÚdeviceÚappendÚindexr’)ržrNÚ device_idsÚidxrr¡r¡r¢r s  ÿý    €zTrainer.device_idscCs t|jƒS)z,Number of devices the trainer uses per node.)Úlenr rùr¡r¡r¢Ú num_devicess zTrainer.num_devicescCrr)rMr¹rùr¡r¡r¢r¹•ózTrainer.lightning_modulecCrr©rMÚ optimizersrùr¡r¡r¢r&šrzTrainer.optimizersÚ new_optimscCs ||j_dSrr%)ržr'r¡r¡r¢r&žs cCrr)rMÚlr_scheduler_configsrùr¡r¡r¢r(¢rzTrainer.lr_scheduler_configscCó |jjjSr)rMrrPrùr¡r¡r¢rP¦s zTrainer.precisioncCst|jddƒS)NÚscaler)rrrùr¡r¡r¢r*ªrzTrainer.scalercCr)zçThe LightningModule, but possibly wrapped into DataParallel or DistributedDataParallel. To access the pure LightningModule, use :meth:`~lightning.pytorch.trainer.trainer.Trainer.lightning_module` instead. )rMr£rùr¡r¡r¢r£®óz Trainer.modelcCsTt|jƒdkrt|jdttfƒs|jdj}n |jdj}n|j}|j  |¡}|S)asThe directory for the current experiment. Use this to save images to, etc... .. note:: You must call this on all processes. Failing to do so will cause your program to stall forever. .. code-block:: python def training_step(self, batch, batch_idx): img = ... save_img(img, self.trainer.log_dir) r) r"Úloggersr‘rrÚsave_dirrrrrMÚ broadcast)ržÚdirpathr¡r¡r¢r¼s  zTrainer.log_dircCr)aWhether this process is the global zero in multi-node training. .. code-block:: python def training_step(self, batch, batch_idx): if self.trainer.is_global_zero: print("in node 0, accelerator 0") )rMÚis_global_zerorùr¡r¡r¢r0Ôs zTrainer.is_global_zerocCst|jtƒr |jjSdSr)r‘rMr#Údistributed_sampler_kwargsrùr¡r¡r¢r1ás z"Trainer.distributed_sampler_kwargscCs&|jjjj ¡otd|jƒo|jdkS)z2Check if we should run validation during training.Úvalidation_stepr)r‰rŠrüÚ _data_sourceÚ is_definedr>r¹rZrùr¡r¡r¢rýçs  ÿýzTrainer.enable_validationcCs&t|jƒrtj tj |j¡¡S|jS)z³The default location to save artifacts of loggers, checkpoints etc. It is used as a fallback if logger or checkpoint callback do not define specific save paths. )rÚ_default_root_dirr~ÚpathÚnormpathÚ expanduserrùr¡r¡r¢rrðs zTrainer.default_root_dircCó|j}t|ƒdkr |dSdS)zThe first :class:`~lightning.pytorch.callbacks.early_stopping.EarlyStopping` callback in the Trainer.callbacks list, or ``None`` if it doesn't exist.rN)Úearly_stopping_callbacksr"©ržrRr¡r¡r¢Úearly_stopping_callbackûózTrainer.early_stopping_callbackcCódd„|jDƒS)zŠA list of all instances of :class:`~lightning.pytorch.callbacks.early_stopping.EarlyStopping` found in the Trainer.callbacks list.cSóg|] }t|tƒr|‘qSr¡)r‘r©Ú.0Úcr¡r¡r¢Ú óz4Trainer.early_stopping_callbacks..©rRrùr¡r¡r¢r:óz Trainer.early_stopping_callbackscCr9)z¡The first :class:`~lightning.pytorch.callbacks.model_checkpoint.ModelCheckpoint` callback in the Trainer.callbacks list, or ``None`` if it doesn't exist.rN)Úcheckpoint_callbacksr"r;r¡r¡r¢Úcheckpoint_callbackr=zTrainer.checkpoint_callbackcCr>)zŽA list of all instances of :class:`~lightning.pytorch.callbacks.model_checkpoint.ModelCheckpoint` found in the Trainer.callbacks list.cSr?r¡)r‘rr@r¡r¡r¢rCrDz0Trainer.checkpoint_callbacks..rErùr¡r¡r¢rGrFzTrainer.checkpoint_callbackscCs"|jD] }t|tƒr|SqdS)z¥An instance of :class:`~lightning.pytorch.callbacks.progress.progress_bar.ProgressBar` found in the Trainer.callbacks list, or ``None`` if one doesn't exist.N)rRr‘r)ržrBr¡r¡r¢Úprogress_bar_callbacks  ÿzTrainer.progress_bar_callbackcCr)aZSet to the path/URL of a checkpoint loaded via :meth:`~lightning.pytorch.trainer.trainer.Trainer.fit`, :meth:`~lightning.pytorch.trainer.trainer.Trainer.validate`, :meth:`~lightning.pytorch.trainer.trainer.Trainer.test`, or :meth:`~lightning.pytorch.trainer.trainer.Trainer.predict`. ``None`` otherwise. )r‡Ú _ckpt_pathrùr¡r¡r¢r¨s zTrainer.ckpt_pathcCs||j_t|ƒ|j_dS)akAllows you to manage which checkpoint is loaded statefully. .. code-block:: python trainer = Trainer() trainer.ckpt_path = "my/checkpoint/file.ckpt" trainer.fit(model) ... # you will be in charge of resetting this trainer.ckpt_path = None trainer.test(model) N)r‡rJÚboolÚ _user_managed)ržr¨r¡r¡r¢r¨*sÚfilepathÚ weights_onlyÚstorage_optionscCs@|jdur tdƒ‚|j |¡}|jj|||d|j d¡dS)a,Runs routine to create a checkpoint. This method needs to be called on all processes in case the selected strategy is handling distributed checkpointing. Args: filepath: Path where checkpoint is saved. weights_only: If ``True``, will only save the model weights. storage_options: parameter for how to save to storage, passed to ``CheckpointIO`` plugin Raises: AttributeError: If the model is not attached to the Trainer before calling this method. NzªSaving a checkpoint is only possible if a model is attached to the Trainer. Did you call `Trainer.save_checkpoint()` before calling `Trainer.{fit,validate,test,predict}`?)rOúTrainer.save_checkpoint)r£ÚAttributeErrorr‡Údump_checkpointrMÚsave_checkpointrï)ržrMrNrOÚ checkpointr¡r¡r¢rS=s ÿ rPcCó|jjtjkSr)r›r­r7Ú INTERRUPTEDrùr¡r¡r¢Ú interrupted\rzTrainer.interruptedcCrUr)r›rër4ÚTRAININGrùr¡r¡r¢r®`rzTrainer.trainingÚvalcCó(|r tj|j_dS|jrd|j_dSdSr)r4rXr›rër®©ržrYr¡r¡r¢r®dó  ÿcCrUr)r›rër4rrùr¡r¡r¢rÇkrzTrainer.testingcCrZr)r4rr›rërÇr[r¡r¡r¢rÇor\cCrUr)r›rër4rÌrùr¡r¡r¢rÍvrzTrainer.predictingcCrZr)r4rÌr›rërÍr[r¡r¡r¢rÍzr\cCrUr)r›rër4r‹rùr¡r¡r¢r¿rzTrainer.validatingcCrZr)r4r‹r›rër¿r[r¡r¡r¢r¿…r\cCs|jjduo |jjjSr)r›rërñrùr¡r¡r¢rñŒszTrainer.evaluatingcCrU)z…Whether sanity checking is running. Useful to disable some hooks, logging or callbacks during the sanity checking. )r›rër4ÚSANITY_CHECKINGrùr¡r¡r¢rÿszTrainer.sanity_checkingcCrZr)r4r]r›rërÿr[r¡r¡r¢rÿ™r\cCr)zyWhether a ``signal.SIGTERM`` signal was received. For example, this can be checked to exit gracefully. )rˆÚreceived_sigtermrùr¡r¡r¢r^ szTrainer.received_sigtermcCr))zƒThe number of optimizer steps taken (does not reset each epoch). This includes multiple optimizers (if enabled). )r‰rŠÚ global_steprùr¡r¡r¢r_­s zTrainer.global_stepcCó |jjjjS)z=The current epoch, updated after the epoch end hooks are run.)r‰Úepoch_progressÚcurrentÚ completedrùr¡r¡r¢Ú current_epoch¶ó zTrainer.current_epochcCrr)r‰rTrùr¡r¡r¢rT»rzTrainer.max_epochscCrr)r‰rUrùr¡r¡r¢rU¿rzTrainer.min_epochscCrr)r‰rVrùr¡r¡r¢rVÃrzTrainer.max_stepscCrr)r‰rWrùr¡r¡r¢rWÇrzTrainer.min_stepscCr`)z,Whether trainer is executing the last batch.)r‰rŠÚbatch_progressÚ is_last_batchrùr¡r¡r¢rgËrezTrainer.is_last_batchcCó|jj}dur |jSdS)z9The training dataloader(s) used during ``trainer.fit()``.N)r‰Ú_combined_loaderÚ iterables©ržÚcombined_loaderr¡r¡r¢Útrain_dataloaderÐózTrainer.train_dataloadercCs.|jjjj}dus|jj}dur|jSdS)zUThe validation dataloader(s) used during ``trainer.fit()`` or ``trainer.validate()``.N)r‰rŠrürirŒrjrkr¡r¡r¢r¦×s  þzTrainer.val_dataloaderscCrh)z6The test dataloader(s) used during ``trainer.test()``.N)rŽrirjrkr¡r¡r¢rÊàrnzTrainer.test_dataloaderscCrh)z?The prediction dataloader(s) used during ``trainer.predict()``.N)rrirjrkr¡r¡r¢rÐçrnzTrainer.predict_dataloaderscCr)zJThe number of training batches that will be used during ``trainer.fit()``.)r‰Ú max_batchesrùr¡r¡r¢Únum_training_batchesîr$zTrainer.num_training_batchescsˆjjjj}‡fdd„|DƒS)zhThe number of validation batches that will be used during the sanity-checking part of ``trainer.fit()``.csg|]}tˆj|ƒ‘qSr¡)Úminr`)rAÚbatchesrùr¡r¢rCøsz2Trainer.num_sanity_val_batches..)r‰rŠrüro)ržror¡rùr¢Únum_sanity_val_batchesós zTrainer.num_sanity_val_batchescCs"|jjtjkr |jjS|jjjj S)zfThe number of validation batches that will be used during ``trainer.fit()`` or ``trainer.validate()``.) r›r«r5r‹rŒror‰rŠrüÚ _max_batchesrùr¡r¡r¢Únum_val_batchesús zTrainer.num_val_batchescCr)zGThe number of test batches that will be used during ``trainer.test()``.)rŽrorùr¡r¡r¢Únum_test_batchesr$zTrainer.num_test_batchescCr)zPThe number of prediction batches that will be used during ``trainer.predict()``.)rrorùr¡r¡r¢Únum_predict_batchesr$zTrainer.num_predict_batchescCsH|jjtjkr |jjjS|jjtjkr|jS|jjtj kr |j St dƒ‚)NzPThe `Trainer._evaluation_loop` property isn't defined. Accessed outside of scope) r›r«r5rªr‰rŠrür‹rŒrrŽrørùr¡r¡r¢rò s zTrainer._evaluation_loopcCs.|jr|jS|js |jr|jS|jr|jSdSr)r®r‰rÿrñròrÍrrùr¡r¡r¢rís zTrainer._active_loopcCst|jƒdkr |jdSdS)zGThe first :class:`~lightning.pytorch.loggers.logger.Logger` being used.rN)r"r,rùr¡r¡r¢rQ%szTrainer.loggercCs|sg|_dS|g|_dSr)r,)ržrQr¡r¡r¢rQ*s  cCs|jS)zÊThe list of :class:`~lightning.pytorch.loggers.logger.Logger` used. .. code-block:: python for logger in trainer.loggers: logger.log_metrics({"foo": 1.0}) ©Ú_loggersrùr¡r¡r¢r,1s zTrainer.loggersr,cCs|r||_dSg|_dSrrx)ržr,r¡r¡r¢r,=scCr)aThe metrics available to callbacks. .. code-block:: python def training_step(self, batch, batch_idx): self.log("a_val", 2.0) callback_metrics = trainer.callback_metrics assert callback_metrics["a_val"] == 2.0 )r…Úcallback_metricsrùr¡r¡r¢rzAszTrainer.callback_metricscCr)zæThe metrics sent to the loggers. This includes metrics logged via :meth:`~lightning.pytorch.core.LightningModule.log` with the :paramref:`~lightning.pytorch.core.LightningModule.log.logger` argument set. )r…Úlogged_metricsrùr¡r¡r¢r{Qr+zTrainer.logged_metricscCr)zíThe metrics sent to the progress bar. This includes metrics logged via :meth:`~lightning.pytorch.core.LightningModule.log` with the :paramref:`~lightning.pytorch.core.LightningModule.log.prog_bar` argument set. )r…Úprogress_bar_metricsrùr¡r¡r¢r|[r+zTrainer.progress_bar_metricscCs|j}|dur |jSdSr)ríÚ_results)ržÚ active_loopr¡r¡r¢r}eszTrainer._resultscCs¢|jdkr|jdkrtdƒS|jS|jdurtdƒ|j ¡|j}|tdƒkr+|jS|jdus2J‚t  ||j ¡t |jdƒ}|jdkrMt ||jƒ}|S|}|S)a?The estimated number of batches that will ``optimizer.step()`` during training. This accounts for gradient accumulation and the current trainer configuration. This might be used when setting up your training dataloader, if it hasn't been set up already. .. code-block:: python def configure_optimizers(self): optimizer = ... stepping_batches = self.trainer.estimated_stepping_batches scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=1e-3, total_steps=stepping_batches) return [optimizer], [scheduler] Raises: MisconfigurationException: If estimated stepping batches cannot be computed due to different `accumulate_grad_batches` at different epochs. rKÚinfNzBLoading `train_dataloader` to estimate number of stepping batches.rJ) rTrVr“rmr?r‰Ú setup_datarpÚmathÚceilreÚmaxrq)ržÚ total_batchesÚmax_estimated_stepsr¡r¡r¢Úestimated_stepping_batchesps    ÿz"Trainer.estimated_stepping_batches)NNNN)NNNTN)NNNNNr)rsN)rsr¤)FN)r|Ú __module__Ú __qualname__r:r Ústrrr$rr’r r)rrrKrrrr“r(r"r rrxrErrDr²r°rBrÂrCrÀrÉrÈrÏrÎrºrèrçrôrÚrrr rrÚpropertyrLrMr!rrrrrrOr r#r¹r r&ÚsetterrFr(r*rPr*rõÚnnÚModuler£rr0r1rýrrrr<r:rrHrGrrIr¨rSrWr®rÇrÍr¿rñrÿr^r_rdrTrUrVrWrgrmr¦rÊrÐrprsrurvrwrròrrrírQr,r0rzr{r1r|r2r}r†Ú __classcell__r¡r¡r r¢rHVsh× ý üûúùø ÷ ö õ ô óòñðïîí ìëêéèçæåäãâá à!ß"Þ#Ý$Ü%Û&Ú'Ù(Ø)×*Ö úþýüûú ù/úþýüûú ù(úþýüûú ùAúþýüûú ù/úþýüûú ùBúþýüûú ù/úþýüûú ùCúþýüûú ù+ÿÿÿ þ a  &      ÿÿÿÿ þ      rH){Ú__doc__Úloggingrr~Ú contextlibrÚdatetimerÚtypingrrrrrr r Úweakrefr rõÚ torch.optimr Úlightning.pytorchÚpytorchÚplÚ%lightning.fabric.utilities.apply_funcr Ú#lightning.fabric.utilities.cloud_iorÚ lightning.fabric.utilities.typesrÚlightning.pytorch.acceleratorsrÚlightning.pytorch.callbacksrrrrÚ!lightning.pytorch.core.datamodulerÚlightning.pytorch.loggersrÚ"lightning.pytorch.loggers.csv_logsrÚ%lightning.pytorch.loggers.tensorboardrÚ#lightning.pytorch.loggers.utilitiesrÚlightning.pytorch.loopsrrÚ'lightning.pytorch.loops.evaluation_looprÚ lightning.pytorch.loops.fit_looprÚ!lightning.pytorch.loops.utilitiesrrÚlightning.pytorch.pluginsr r!Úlightning.pytorch.profilersr"Úlightning.pytorch.strategiesr#r$Úlightning.pytorch.trainerr%r&Ú1lightning.pytorch.trainer.configuration_validatorr'Ú:lightning.pytorch.trainer.connectors.accelerator_connectorr(r)r*r+Ú7lightning.pytorch.trainer.connectors.callback_connectorr,Ú9lightning.pytorch.trainer.connectors.checkpoint_connectorr-Ú3lightning.pytorch.trainer.connectors.data_connectorr.Ú5lightning.pytorch.trainer.connectors.logger_connectorr/ÚÚ%lightning.pytorch.utilities.rank_zeror?r@Ú lightning.pytorch.utilities.seedrAÚ!lightning.pytorch.utilities.typesrBrCrDrErFÚ$lightning.pytorch.utilities.warningsrGÚ getLoggerr|ryrHr¡r¡r¡r¢Úsb  $