o zig@sddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZmZmZddlZddlmZddlmZddlmZmZdd lmZddlZ dd l!m"Z"dd l#m$Z$dd l%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ddlm?Z?ddl@mAZAmBZBmCZCddlDmEZEeFeGZHeAZIe rddlJZJdejKjddfddZLGddde7ZMdS) N) OrderedDict)contextmanager)Path) TYPE_CHECKINGAnyDict GeneratorListMappingOptionalTupleUnion)Module) Optimizer) LRSchedulerReduceLROnPlateau)override)ClusterEnvironment)_StrategyRegistry)_DEEPSPEED_AVAILABLE_format_precision_config_validate_checkpoint_directory _validate_device_index_selection)_optimizers_to_device) reset_seed)_PATH)CUDAAccelerator)"_init_optimizers_and_lr_schedulers) Precision) DDPStrategy TrainerFn)GradClipAlgorithmType)MisconfigurationException) is_overridden) WarningCacherank_zero_inforank_zero_warn)LRSchedulerConfigmodelreturncCs@|D]}t|_d|_t|_t|_t|_t|_qdSN)modulesr_backward_hooks_is_full_backward_hook_forward_hooks_forward_pre_hooks_state_dict_hooks_load_state_dict_pre_hooks)r)moduler4Z/home/ubuntu/.local/lib/python3.10/site-packages/pytorch_lightning/strategies/deepspeed.pyremove_module_hooks<s  r6c[seZdZdZdZddddddddd d d dd d dddddddddddddddejddddddddddddddf,deddede dee dedede de de d e d!e d"e d#e d$e d%e d&ed'ed(e d)ed*e d+ed,ed-ed.ed/e d0e d1ed2e e e fd3ee e e e effd4e d5eeejd6eed7ed8e d9e d:e d;e ded?ed@edAeedBee dCdfZfdDdE ZedfdFdG ZeddHdIZeddLdMZddNdOZddPdQZeedCefdRdSZedTedUeedCe dVeeffdWdXZ! ddTedYeedZee e"e#fdCe dVeffd[d\Z$dd]d^Z%dCe eee&ffd_d`Z'edCefdadbZ(dTedCdfdcddZ)e*eddeeedCe+dfffdgdh Z,e*edCe+dffdidjZ-ddkdlZ.dTedCdfdmdnZ/eedCe e e ffdodpZ0eddqdrZ1dTedCefdsdtZ2eedCefdudvZ3eddwdxZ4edCefdydzZ5edd{e d|e d}eedCdfd~dZ6ede dCe e efffdd Z7eedCefddZ8edd{e9e efdedCdfddZ:de9e efdedCdfddZ;ed{e9e efdCdfddZdCdfddZ?d3ee e e e effdCee e effddZ@dddZAdddZBded1ed2e e e fded?ededede de de d e d!e d"e d#e d)ed$e d%e d&ed'ed(e dedCe f0ddZCdddZDdCe fddZEZFS)DeepSpeedStrategy deepspeedPL_DEEPSPEED_CONFIG_PATHNTFcpuz /local_nvmeiiʚ;ilJ)i autori acceleratorzpl.accelerators.Acceleratorzero_optimizationstage remote_deviceoffload_optimizeroffload_parametersoffload_params_device nvme_pathparams_buffer_countparams_buffer_size max_in_cpuoffload_optimizer_deviceoptimizer_buffer_count block_size queue_depth single_submitoverlap_events thread_count pin_memorysub_group_sizecontiguous_gradients overlap_commallgather_partitionsreduce_scatterallgather_bucket_sizereduce_bucket_sizezero_allow_untested_optimizerlogging_batch_size_per_gpuconfig logging_levelparallel_devicescluster_environment loss_scaleinitial_scale_powerloss_scale_window hysteresismin_loss_scalepartition_activationscpu_checkpointingcontiguous_memory_optimizationsynchronize_checkpoint_boundaryload_full_weightsprecision_pluginprocess_group_backendr*c-.s:tstdtj||| |+|,d|||_|jdurw|j|||fid|d|d|d|d| d | d | d |d | d | d|d|d|d|d|d|&d|'d|(d|)d|d|d|d|d|d|d|d||_ddl}-d |_|-j j j |||_ |*|_|!|_|"|_|#|_|$|_|%|_dS)!aProvides capabilities to run training using the DeepSpeed library, with training optimizations for large billion parameter models. `For more information: https://pytorch- lightning.readthedocs.io/en/stable/advanced/model_parallel.html#deepspeed`. .. warning:: This is an :ref:`experimental ` feature. Defaults have been set to enable ZeRO-Offload and some have been taken from the link below. These defaults have been set generally, but may require tuning for optimum performance based on your model size. `For more information: https://www.deepspeed.ai/docs/config-json/#zero-optimizations-for-fp16-training`. Arguments: zero_optimization: Enable ZeRO optimization. This is compatible with either `precision="16-mixed"` or `precision="bf16-mixed"`. stage: Different stages of the ZeRO Optimizer. 0 is disabled, 1 is optimizer state partitioning, 2 is optimizer+gradient state partitioning, 3 is optimizer+gradient_parameter partitioning using the infinity engine. remote_device: Device to instantiate the model on initially (``cpu`` or ``nvme``). Defaults to GPU. offload_optimizer: Enable offloading optimizer memory and computation to CPU or NVMe based on ``offload_optimizer_device``. offload_parameters: When using ZeRO Stage 3, Enable offloading parameter memory and computation to CPU or NVMe based on ``offload_params_device``. offload_params_device: When offloading parameters choose the device to offload to, ``cpu`` or ``nvme``. offload_optimizer_device: When offloading optimizer state choose the device to offload to, ``cpu`` or ``nvme``. params_buffer_count: Number of buffers in buffer pool for parameter offloading when ``offload_params_device`` is ``nvme``. params_buffer_size: Size of buffers in buffer pool for parameter offloading when ``offload_params_device`` is ``nvme``. max_in_cpu: Number of parameter elements to maintain in CPU memory when offloading to NVMe is enabled. nvme_path: Filesystem path for NVMe device for optimizer/parameter state offloading. optimizer_buffer_count: Number of buffers in buffer pool for optimizer state offloading when ``offload_optimizer_device`` is set to to ``nvme``. This should be at least the number of states maintained per parameter by the optimizer. For example, Adam optimizer has 4 states (parameter, gradient, momentum, and variance). block_size: When using NVMe Offloading, the I/O block size in bytes. queue_depth: When using NVMe Offloading, the I/O queue depth. single_submit: When using NVMe Offloading, submit requests to storage device as multiple individual requests, as opposed to one block of requests. overlap_events: When using NVMe Offloading, submit requests to storage device in an overlapped fashion without waiting for completion of earlier requests. thread_count: When using NVMe Offloading, Intra-request parallelism for each read/write submitted by a user thread. pin_memory: When using ZeRO stage 3, pin optimizer state memory on CPU. This could boost throughput at the cost of extra memory overhead. sub_group_size: When using ZeRO stage 3, defines the number of parameters within a sub group to offload at a time. Smaller numbers require more communication, but improve memory efficiency. contiguous_gradients: Copies gradients to a continuous buffer as they are produced. Avoids memory fragmentation during backwards. Useful when training large models. overlap_comm: Overlap the reduction (synchronization) of gradients with the backwards computation. This is a speed optimization when training across multiple GPUs/machines. allgather_partitions: All gather updated parameters at the end of training step, instead of using a series of broadcast collectives. reduce_scatter: Use reduce/scatter instead of allreduce to average gradients. allgather_bucket_size: Number of elements to allgather at once. Used to limit the memory required for larger model sizes, with a tradeoff with speed. reduce_bucket_size: Number of elements to reduce at once. Used to limit the memory required for larger model sizes, with a tradeoff with speed. zero_allow_untested_optimizer: Allow untested optimizers to be used with ZeRO. Currently only Adam is a DeepSpeed supported optimizer when using ZeRO. logging_batch_size_per_gpu: Config used in DeepSpeed to calculate verbose timing for logging on a per sample per second basis (only displayed if logging=logging.INFO). If set to "auto", the strategy tries to infer this from the train DataLoader's BatchSampler, else defaults to 1. To obtain accurate logs when using datasets that do not support batch samplers, set this to the actual per gpu batch size (trainer.batch_size). config: Pass in a deepspeed formatted config dict, or path to a deepspeed config: https://www.deepspeed.ai/docs/config-json. All defaults will be ignored if a config is passed in. logging_level: Set logging level for deepspeed. loss_scale: Loss scaling value for FP16 training. 0.0 results in dynamic loss scaling, otherwise static. initial_scale_power: Power of the initial dynamic loss scale value. Loss scale is computed by ``2^initial_scale_power``. loss_scale_window: Window in which to raise/lower the dynamic FP16 loss scaling value. hysteresis: FP16 Delay shift in Dynamic Loss scaling. min_loss_scale: The minimum FP16 dynamic loss scaling value. partition_activations: Enables partition activation when used with ZeRO stage 3 and model parallelism. Still requires you to wrap your forward functions in deepspeed.checkpointing.checkpoint. See `deepspeed tutorial `_. cpu_checkpointing: Offloads partitioned activations to CPU if ``partition_activations`` is enabled. contiguous_memory_optimization: Copies partitioned activations so that they are contiguous in memory. Not supported by all models. synchronize_checkpoint_boundary: Insert :func:`torch.cuda.synchronize` at each checkpoint boundary. load_full_weights: True when loading a single checkpoint file containing the model state dict when using ZeRO Stage 3. This differs from the DeepSpeed checkpoint which contains shards per worker. ztTo use the `DeepSpeedStrategy`, you must have DeepSpeed installed. Install it by running `pip install -U deepspeed`.)rBr`rarlrmNrFrGrIrHrJrKrLrTrMrNrOrPrQrRrSrgrhrirjrDrVrWrXrYrZr[rUrF)rr#super__init__ _load_configr^_create_default_configr8_config_initializedutilsloggingloggersetLevelrErkrbrcrdrerf).selfrBrCrDrErFrGrHrIrJrKrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmr8 __class__r4r5roKs3         zDeepSpeedStrategy.__init__cs0t|jtstd|jjjdtdS)Nz;The DeepSpeed strategy is only supported on CUDA GPUs but `z ` is used.) isinstancerBr RuntimeErrorry__name__rnsetup_environmentrwrxr4r5r}=s z#DeepSpeedStrategy.setup_environmentcCs2|jdusJt|jt||dSr+)r`rrset_world_ranks_init_deepspeed_distributedr~r4r4r5setup_distributedFs   z#DeepSpeedStrategy.setup_distributedtrainer pl.TrainercCs||jdus J|j||jdusJ|j|j|_||j|_|jjt j kr3| || |jjt j krEt |j|j||dSr+)_init_config_if_neededrBsetupr)rlconvert_module _setup_modelstatefnr!FITTINGsetup_optimizerssetup_precision_pluginr optimizers root_deviceinit_deepspeedbarrierrwrr4r4r5rNs   zDeepSpeedStrategy.setupcCsrddl}|jdus Jtdkr(|td|jd|jdd|j| |_ |j |j |jj ddS)NrWindowsz1initializing deepspeed distributed: GLOBAL_RANK: z , MEMBER: r?/)distributed_port) r8raplatformsystem_set_node_environment_variablesloginfo global_rank world_size_get_process_group_backend_process_group_backendinit_distributed main_portrwr8r4r4r5ras  z-DeepSpeedStrategy._init_deepspeed_distributedcCsb|jdusJ|jjtjd<t|jjtjd<t|jtjd<t|jtjd<t|jtjd<dS)N MASTER_ADDR MASTER_PORTRANK WORLD_SIZE LOCAL_RANK) ra main_addressosenvironstrrrr local_rankr~r4r4r5rps z1DeepSpeedStrategy._set_node_environment_variablescCdSNTr4r~r4r4r5restore_checkpoint_after_setupxz0DeepSpeedStrategy.restore_checkpoint_after_setupr)rdeepspeed.DeepSpeedEnginecCsft|dkrtdt|d|jdusJ|jdd|||d\|_}||j|gfS)aSetup a model and multiple optimizers together. Currently only a single optimizer is supported. Return: The model wrapped into a :class:`deepspeed.DeepSpeedEngine` and a list with a single deepspeed optimizer. r?z>Currently only one optimizer is supported with DeepSpeed. Got z optimizers instead.Ntrain_micro_batch_size_per_gpur)len ValueErrorr^ setdefault_setup_model_and_optimizerr)'_set_deepspeed_activation_checkpointing)rwr)r optimizerr4r4r5_setup_model_and_optimizers}s  z-DeepSpeedStrategy._setup_model_and_optimizersr lr_schedulerc CsPddl}tdd|}|jtj|jjd|j||||dd\}}}}||fS)zInitialize one model and one optimizer with an optional learning rate scheduler. This calls :func:`deepspeed.initialize` internally. rNcS|jSr+) requires_grad)pr4r4r5sz>DeepSpeedStrategy._setup_model_and_optimizer.. device_rankF)argsr^r)model_parametersrrdist_init_required) r8filter parameters initializeargparse Namespacerindexr^) rwr)rrr8rdeepspeed_enginedeepspeed_optimizer_r4r4r5rs  z,DeepSpeedStrategy._setup_model_and_optimizercCs|jdusJtd|jtjrtd|jjjtjkrt dt |j tjs(J|jjr9|jjj r9| |j dS||j dS)Nconfigure_gradient_clippinga8Since DeepSpeed handles gradient clipping internally, the default `LightningModule.configure_gradient_clipping` implementation will not actually clip gradients. The hook will still be called. Consider setting `Trainer(gradient_clip_val=..., gradient_clip_algorithm='norm')` which will use the internal mechanism.z7DeepSpeed does not support clipping gradients by value.)lightning_moduler$plLightningModuler'rgradient_clip_algorithmr"VALUEr#rzr)training_initialize_deepspeed_train_initialize_deepspeed_inferencer~r4r4r5rsz DeepSpeedStrategy.init_deepspeedcCsV|jdusJt|j\}}t|dkst|dkrtd|d|r(|dfSdfS)Nr?zNDeepSpeed currently only supports single optimizer, single optional scheduler.r)rrrr#)rwr lr_schedulersr4r4r5_init_optimizerssz"DeepSpeedStrategy._init_optimizerscCs2t|jtsJ|jd}|duo|ddkS)NrCrD)rzr^dictget)rwrCr4r4r5 zero_stage_3s zDeepSpeedStrategy.zero_stage_3cCsd\}}t|jts Jd|jvrtdd}n |\}}|dur%|j}||||\}}||g|_|j }|durRd|_ |durKt |dd}n||_|g|_ ||_ dS)N)NNrzYou have specified an optimizer and/or scheduler within the DeepSpeed config. It is recommended to define it in `LightningModule.configure_optimizers`.step)interval) rzr^rr&r schedulerrrrrr(lr_scheduler_configsr))rwr)rrrrdeepspeed_schedulerr4r4r5rs0  z-DeepSpeedStrategy._initialize_deepspeed_train empty_init)NNNc#sd|jr|durtd|ddVdStj|d dVWddS1s+wYdS)NFz `empty_init=zN` is not a valid choice with `DeepSpeedStrategy` when ZeRO stage 3 is enabled.)r)rNotImplementedErrorrntensor_init_context)rwrrxr4r5rs "z%DeepSpeedStrategy.tensor_init_contextccsVddl}||jj|j|j|jd dVWddS1s$wYdS)Nr)enabledrEconfig_dict_or_path)r8rzeroInitrrEr^rr4r4r5model_sharded_contexts"z'DeepSpeedStrategy.model_sharded_contextcCsdddl}t|jts J|jdr0|jd}|jjd|d|d|d|dddSdS)Nractivation_checkpointingrgrirhprofile)mpu_rgcontiguous_checkpointingcheckpoint_in_cpur)r8rzr^rr checkpointing configure)rwr8checkpoint_configr4r4r5rs   z9DeepSpeedStrategy._set_deepspeed_activation_checkpointingc Csddl}t|jts Jddi}d|jvr|d|jdid|jvr.|d|jdi|jr?||jd|jddt||jtj |j j d ||ddgd d \}}}}||_ dS) Nrrr?fp16bf16r\rCr\rCrF)rr^r)rrrr) r8rzr^rupdaterr6rrrrrr))rwr)r8inference_configrr4r4r5r s.   z1DeepSpeedStrategy._initialize_deepspeed_inferencecCs|j|jdS)N) num_replicasrank)rrr~r4r4r5distributed_sampler_kwargs=sz,DeepSpeedStrategy.distributed_sampler_kwargscCsg|_g|_dS)zCreates optimizers and schedulers. Args: trainer: the Trainer, these optimizers should be connected to N)rrrr4r4r5rBs z"DeepSpeedStrategy.setup_optimizerscCs|Sr+r4)rwr)r4r4r5rQszDeepSpeedStrategy._setup_modelcCr)z>Whether the strategy handles gradient accumulation internally.Tr4r~r4r4r5handles_gradient_accumulationTsz/DeepSpeedStrategy.handles_gradient_accumulationcCrr+)r)r~r4r4r5rZsz"DeepSpeedStrategy.deepspeed_enginecCs|jdkp |jdkS)Nr?) num_processes num_nodesr~r4r4r5 _multi_device^szDeepSpeedStrategy._multi_device checkpointfilepathstorage_optionscsv||}|durtd|jjd|jr!|jr!|jr!tdddgfdd| D}|j j ||d d dS) aSave model/training states as a checkpoint file through state-dump and file-write. Args: checkpoint: The checkpoint state dictionary filepath: write-target file's path storage_options: not used for ``DeepSpeedStrategy`` as ``CheckpointIO`` is not used Raises: TypeError: If ``storage_options`` arg is passed in Nze`Trainer.save_checkpoint(..., storage_options=...)` with `storage_options` arg is not supported for `z ` as `CheckpointIO` is not used.aWhen saving the DeepSpeed Stage 3 checkpoint, each worker will save a shard of the checkpoint within a directory. If a single file is required after training, see https://lightning.ai/docs/pytorch/stable/advanced/model_parallel.html#deepspeed-zero-stage-3-single-file for instructions. state_dictoptimizer_statescsi|] \}}|vr||qSr4r4).0kv _exclude_keysr4r5 sz5DeepSpeedStrategy.save_checkpoint..r) client_statetag) broadcast TypeErrorryr|rris_global_zero warning_cachewarnitemsrsave_checkpoint)rwrrrr4rr5rbs  z!DeepSpeedStrategy.save_checkpointcheckpoint_pathcs|jr|jr||}t|St||jdusJddlm}|jj j j |j k}|j j||d|jjd\}}|durAtd|S)Nrr F)load_optimizer_statesload_lr_scheduler_statesload_module_strictzDeepSpeed was unable to load the checkpoint. Ensure you passed in a DeepSpeed compatible checkpoint or a single checkpoint file with `Trainer(strategy=DeepSpeedStrategy(load_full_weights=True))`.)rkrrrnload_checkpointrr pytorch_lightning.trainer.statesr!rrrrrstrict_loadingr#)rwrr! is_fittingrrrxr4r5rs$     z!DeepSpeedStrategy.load_checkpointcCs8|jdusJ|jr|jr|jjjjtjkrtddS)NaA single checkpoint file has been given. This means optimizer states cannot be restored. If you'd like to restore these states, you must provide a path to the originally saved DeepSpeed checkpoint. When using ZeRO 3, the original path should be a directory.F) rrkrrrrr!rr'r~r4r4r5lightning_restore_optimizers z-DeepSpeedStrategy.lightning_restore_optimizerstrictcCs.|jr|jr||j||ddSdSdS)N)r)rkrmodel_to_device_restore_zero_state)rwrrr4r4r5load_model_state_dicts z'DeepSpeedStrategy.load_model_state_dictckptcsPddljdus Jd dtjjdtddffdd jdd dS) a8Overrides the normal load_state_dict behaviour in PyTorch to ensure we gather parameters that may be sharded across processes before loading the state dictionary when using ZeRO stage 3. This is then automatically synced across processes. Args: ckpt: The ckpt file. rNr3prefixr*c sg}g}g}d}t|dd}|}|dur||_|dur!in ||ddi}jjt|jddddjrI|j ||||||dWdn1sSwY|j D]\}} | durn| ||d q]dS) Nr _metadataF)recurser) modifier_rank)rrlocal_metadatar missing_keysunexpected_keys error_msgs.) getattrcopyrrrGatheredParameterslistrr _load_from_state_dict_modulesr ) r3rr$r%r&rmetadatar#namechildrr8loadrwrr4r5r2s6   z3DeepSpeedStrategy._restore_zero_state..load)r)r)r8rtorchnnrr)rwrrr4r1r5rs ( z%DeepSpeedStrategy._restore_zero_statecCsdSr+r4)rwrr4r4r5load_optimizer_state_dictrz+DeepSpeedStrategy.load_optimizer_state_dictstrategy_registryc Cs|jd|dd|jd|ddd|jd|d d d|jd |d d d d|jd|ddd|jd|ddd d d|jd|ddd d dddd dS)Nr8zDefault DeepSpeed Strategy) descriptiondeepspeed_stage_1z#DeepSpeed with ZeRO Stage 1 enabledr?)r7rDdeepspeed_stage_2z#DeepSpeed with ZeRO Stage 2 enabledr:deepspeed_stage_2_offloadz&DeepSpeed ZeRO Stage 2 and CPU OffloadT)r7rDrFdeepspeed_stage_3zDeepSpeed ZeRO Stage 3rdeepspeed_stage_3_offloadz&DeepSpeed ZeRO Stage 3 and CPU Offload)r7rDrFrGdeepspeed_stage_3_offload_nvmez'DeepSpeed ZeRO Stage 3 and NVMe Offloadnvme)r7rDrFrGrErHrM)register)clsr6r4r4r5register_strategiess< z%DeepSpeedStrategy.register_strategiescCs|dur|jtjvrtd|jdtj|j}t|ttfrFtj|s-t d|t | }t |}Wdn1sAwYt|t sQ|dusQJ|S)Nz"Loading DeepSpeed config from set z environment variablezHYou passed in a path to a DeepSpeed config but the path does not exist: )DEEPSPEED_ENV_VARrrr&rzrrpathisfiler#openjsonr2r)rwr^fr4r4r5rps    zDeepSpeedStrategy._load_configcCs|js |d|_dSdSr)rr_format_configr~r4r4r5rs z(DeepSpeedStrategy._init_config_if_neededc CsD|jdur td|t|j|jj|j|j|j|j |j ddS)NzTo use DeepSpeed you must pass in a DeepSpeed config dict, or a path to a JSON config. See: https://lightning.ai/docs/pytorch/stable/advanced/model_parallel.html#deepspeed)r^ precisionrbrdrfrcre) r^r#(_format_batch_size_and_grad_accum_configrrlrIrbrdrfrcrer~r4r4r5rH"s  z DeepSpeedStrategy._format_config zero_kwargscKs~||||d|||||dd}|r3|}|r|| ||d|d<| r,| | | | ||d|d<||d|}|d kr=d |i|}|S) N)rgrhrirj)rOrPrQrRrS)raio)devicerI buffer_countrTrF)rMrIrN buffer_sizerLrT offload_paramrr@rr4)rwrCr\r]rgrhrirjrFrGrIrHrJrKrLrMrNrTrOrPrQrRrSrKcfg zero_configr4r4r5rq3sH    z(DeepSpeedStrategy._create_default_configcCst|jtsJ|jdurdSd|jvrtd|jjj|jd<d|jvr.|}||jd<d|jvr?|jjjp9d|jd<dSdS)Ngradient_accumulation_stepszDo not set `gradient_accumulation_steps` in the DeepSpeed config as this will be set with the `accumulate_grad_batches` argument passed via the Lightning Trainer.rgradient_clippingg) rzr^rrr#raccumulate_grad_batches_auto_select_batch_sizegradient_clip_val)rw batch_sizer4r4r5rJxs     z:DeepSpeedStrategy._format_batch_size_and_grad_accum_configcCsD|jdusJd}|jjjj}|r |}t|dr |jj}|S)Nr? batch_sampler) rrfit_loop _data_source is_defined dataloaderhasattrrYrX)rwrX data_sourcetrain_dataloaderr4r4r5rVs  z)DeepSpeedStrategy._auto_select_batch_size)r*N)rrr*Nr+)r*r)T)Gr| __module__ __qualname__ strategy_namerBrtWARNr boolintrr rrrr r3rMrfloatrrorr}rrrrpropertyrrrr rrrrrr(rrrrrrrrrrrrrrrrrrr rrr5 classmethodrrArprrHrqrJrV __classcell__r4r4rxr5r7Gs       !"#$%&'()*+,-.s        "$    "$  "/2         Er7)NrrFrtrr collectionsr contextlibrpathlibrtypingrrrrr r r r r r3torch.nnr torch.optimrtorch.optim.lr_schedulerrrtyping_extensionsrpytorch_lightningrlightning_fabric.pluginsrlightning_fabric.strategiesr%lightning_fabric.strategies.deepspeedrrrr$lightning_fabric.utilities.optimizerrlightning_fabric.utilities.seedr lightning_fabric.utilities.typesr#pytorch_lightning.accelerators.cudar pytorch_lightning.core.optimizerr#pytorch_lightning.plugins.precisionr pytorch_lightning.strategies.ddprrr!pytorch_lightning.utilitiesr"&pytorch_lightning.utilities.exceptionsr#)pytorch_lightning.utilities.model_helpersr$%pytorch_lightning.utilities.rank_zeror%r&r'!pytorch_lightning.utilities.typesr( getLoggerr|rr r8r4r6r7r4r4r4r5sJ    ,