o 2wie1@s:dZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZddlmZddlZd d lmZd d lmZmZd d lmZmZd#dejeejfdefddZGddde Z Gddde Z!Gddde Z"GdddZ#d d d dddZ$GdddZ%dd Z&Gd!d"d"e Z'dS)$z Support for PyTorch lightning. You should just replace the call to `Trainer(...)` with `get_trainer(...)`. For using `dora.log.LogProgress` as a progress bar with PL, see `PLLogProgress`. N)LightningModule)Callback)ProgressBarBase)ClusterEnvironment)Trainer)from_argparse_args)distrib)get_xpis_xp)bold LogProgressTmetricsepochcCsi}|D]:\}}|r|drq|s|drq|ds$|dr,|ddd}t|tjr<|dkr<|}|||<q|S)aFilters metrics before formatting, in particular to remove the `_step` or `_epoch` suffix. This will also convert torch tensors to float. Args: metrics: dict given by PL. epoch: if True, keep only epoch level metrics, otherwise, keep only step level metrics. _step_epoch_rr)itemsendswithrsplit isinstancetorchTensornumelitem)rroutkeyvaluerK/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/dora/lightning.py_filter_metricss r cseZdZfddZdefddZedefddZdefdd Z d edd fd d Z defddZ dedd fddZ defddZ defddZedefddZedefddZedefddZZS)DoraEnvironmentcs tt|_tdSN)super__init__r get_distrib_specspecset_distrib_envself __class__rrr$4s   zDoraEnvironment.__init__returncCdSNTrr(rrrcreates_children9z DoraEnvironment.creates_childrencCr-r.rr(rrrcreates_processes_externally<z,DoraEnvironment.creates_processes_externallycC|jjSr")r& world_sizer(rrrr4@zDoraEnvironment.world_sizesizeNcCdSr"r)r)r6rrrset_world_sizeCr0zDoraEnvironment.set_world_sizecCr3r")r&rankr(rrr global_rankFr5zDoraEnvironment.global_rankr9cCr7r"r)r)r9rrrset_global_rankIr0zDoraEnvironment.set_global_rankcCr3r")r& local_rankr(rrrr<Lr5zDoraEnvironment.local_rankcCr3r")r& node_rankr(rrrr=Or5zDoraEnvironment.node_rankcCr-NFrrrrrdetectRr2zDoraEnvironment.detectcCs tjdS)N MAIN_ADDR)osenvironr(rrr main_addressVs zDoraEnvironment.main_addresscCsttjdS)N MAIN_PORT)intrArBr(rrr main_portZszDoraEnvironment.main_port)__name__ __module__ __qualname__r$boolr/propertyr1rEr4r8r:r;r<r= staticmethodr?strrCrF __classcell__rrr*rr!3s" r!c@s(eZdZdZddZddZddZdS) DoraCheckpointSyncz>Make sure Dora history, and checkpoint state are in sync. cCs t|_dSr")r xpr(rrrr$bs zDoraCheckpointSync.__init__cCs|d}|jj|dS)Ndora_link_history)rPlinkupdate_history)r)trainer pl_module checkpointhistoryrrron_load_checkpointesz%DoraCheckpointSync.on_load_checkpointcCs*|jjj|d<|jj|d<|jj|d<|S)NrQdora_sigdora_cfg)rPrRrWsigcfgr)rTrUrVrrron_save_checkpointis  z%DoraCheckpointSync.on_save_checkpointN)rGrHrI__doc__r$rXr^rrrrrO_s  rOcs8eZdZdZfddZddZddZdd ZZS) DoraHistoryLoggerz,Save metrics to Dora using the XP link. csttj|_dSr")r#r$r rRr(r*rrr$s zDoraHistoryLogger.__init__cC d|_dSr. _first_validr)rTrUrrr on_fit_startw zDoraHistoryLogger.on_fit_startcCrbr>rcrerrron_train_epoch_startzrgz&DoraHistoryLogger.on_train_epoch_startcCs,|jrdS|j}t|dd}|j|dS)NTr)rdlogged_metricsr rR push_metrics)r)rTrUrrrr on_epoch_end}s  zDoraHistoryLogger.on_epoch_end) rGrHrIr_r$rfrhrlrNrrr*rr`ps  r`c@seZdZddZdS)_DummySLURMConnectorcCr7r"rr(rrrregister_slurm_signal_handlersr0z3_DummySLURMConnector.register_slurm_signal_handlersN)rGrHrIrnrrrrrms rm) auto_resumeadd_dora_loggerno_unfinished_epochscOsxtstdtj}t|dr|j}t|dstj|dgt|fi|}|d=| dp/g}t }t t j |}|dkrI||dg7}||d<| ddpTg} | t| |d<|d duritd |d dkrstd ||d <|jj|d <tj|d <|r|dt|d} |r| durtjd} | rt| } nd} | |d<tdi|} |rt| _| S)aReturn a PL trainer, adding the necessary glue code to make everything works. The arguments are exactly the same as for `pytorch_lightning.trainer.Trainer`, with a few extras documented after. ..note:: You should not pass `gpus=` or `num_nodes=` arguments as those will be filled by Dora. Args: auto_resume (bool): if True, automatically resume previous checkpoints. You are still responsible for creating the `ModelCheckpoint` callback, this only handles the `resume_from_checkpoint` part. add_dora_logger (bool): if True, adds a Dora Logger to automatically forward the metrics (those logged with per_epoch=True), otherwise pushing metrics will be up to you. no_unfinished_epochs (bool): if True, deactivates SLURM signal handling by PL, which can result in half finished epoch with each interruption. It is recommended to instead dump a checkpoint every epoch and resume from that one so that training is reliable. z.This can only be called from inside a Dora XP. __wrapped__Nr)pluginsrddp callbacksgpuszCYou cannot specify the number of GPUs, as this is provided by Dora. num_nodeszDYou cannot specify the number of nodes, as this is provided by Dora.default_root_dirresume_from_checkpointz last.ckptr)r RuntimeErrorrr$hasattrrrinspect getcallargslistpopr!minrcuda device_countr4appendrOr&rwr folderr`getis_filerMrmslurm_connector)rorprqargskwargsinitrsenvrvrurylastresumerTrrr get_trainersJ             rc@s eZdZeejddZdS) _InterceptcOs||_||_dSr")rrr)rrrrrr$s z_Intercept.__init__N)rGrHrI functoolswrapsrr$rrrrrs rcKs$tt|fi|}t|ji|jSr")rrrrr)rr interceptrrrtrainer_from_argparse_argssrcseZdZdZd0fdd Zdeddffdd Zfd d Zede fd d Z d1de j ee j fdedefddZeddZddZfddZfddZddZfddZfddZd d!Zd"d#Zfd$d%Zfd&d'Zfd(d)Zd*d+Zd,d-Zd.d/ZZS)2 PLLogProgressz<`dora.log.LogProgress` support for Pytorch-Lightning. r,Nc s t||_||_d|_dSr")r#r$loggerr _pl_module)r)rrr*rrr$s  zPLLogProgress.__init__stagecs t|||||_g|_dSr")r#setupr_replay_history)r)rTrUrr*rrrs zPLLogProgress.setupcst||d|_d|_dS)NFT)r#rf _in_trainrdrer*rrrfs zPLLogProgress.on_fit_startcCs|jdusJ|jSr")rr(rrrrUszPLLogProgress.pl_moduleFrrcCs2i}|D]\}}t|trt|d||<q|S)agDefault method to format metrics for displaying in the progress bar. To customize, you can define a `format_metrics()` method on your Lightning module. Args: metrics: dict of metrics given by PL. stage: "train" or "valid". epoch: if True, provided metrics are for the end of epoch summary. z.5f)rrfloatformat)r)rrrrrrrrrformat_metricss   zPLLogProgress.format_metricscCst|jd|jS)Nr)getattrrUrr(rrr_format_metrics szPLLogProgress._format_metricscCs|jd|j|dkrdnd|d|jjd}|dkr)t|j}n|dkr3t|j}ntd|t |}t |j|f||d |j |_ t |j dS) NzF----------------------------------------------------------------------trainz Training...z Validating...z | Epoch rvalidzInvalid stage )totalname)rinfo capitalizerT current_epochrEtotal_train_batchestotal_val_batchesrzranger rlogprogiter)r)rrrloaderrrr_on_epoch_starts   zPLLogProgress._on_epoch_startcs$|dd|_d|_t||S)NrTF)rrrdr#rhrer*rrrhs z"PLLogProgress.on_train_epoch_startcs|dt||SNr)rr#on_validation_epoch_startrer*rrr$raz'PLLogProgress.on_validation_epoch_startcCsL||j|j}t|dd}|j||dd}|jjdi|t|jdS)NFrir) get_metricsrTrUr rrupdatenext)r)rr formattedrrr _on_batch_end(s  zPLLogProgress._on_batch_endc tj|i||ddS)Nr)r#on_train_batch_endrrr*rrr/z PLLogProgress.on_train_batch_endcrr)r#on_validation_batch_endrrr*rrr3rz%PLLogProgress.on_validation_batch_endcCsZ|dkr|jjjjdd}n |jjjjjdd}t|dd}|||jj|dS)NrFlogri) rTfit_loop epoch_loop_resultsrval_loopr _show_epoch_summaryr)r)rrrrr _on_stage_end7s  zPLLogProgress._on_stage_endcCsj|j|||f|j||dd}|}ddd|D}|jt|d|dd|dS)NTriz | css&|]\}}|d|VqdS)=N)r).0rvalrrr Ds z4PLLogProgress._show_epoch_summary..z Summary | End of Epoch r) rrrrjoinrrrr )r)rrrrrsummaryrrrr@s (z!PLLogProgress._show_epoch_summarycs<t|||js|jsJ|js|dd|_dSdSNrF)r# on_train_endrrdrrer*rron_validation_startIs   z!PLLogProgress.on_validation_startcs(t|||jr|dd|_dSr)r#rlrrrer*rrrlPs  zPLLogProgress.on_epoch_endcst|||ddSr)r#on_validation_endrrer*rrrVszPLLogProgress.on_validation_endcCr7r"rr(rrrdisableZszPLLogProgress.disablecCs4|dg}|r|jd|D]}|j|qdS)Ndora_replay_historyzReplaying past metrics...)rrrr)r)rTrUrVreplay_historysteprrrrX_s   z PLLogProgress.on_load_checkpointcCs|j|d<|S)Nr)rr]rrrr^fs z PLLogProgress.on_save_checkpoint)r,N)F) rGrHrIr_r$rMrrfrKrrUtpDictAnyrJrrrrhrrrrrrrrlrrrXr^rNrrr*rrs:         r)T)(r_rr|rAtypingrpytorch_lightningrpytorch_lightning.callbacksr$pytorch_lightning.callbacks.progressr&pytorch_lightning.plugins.environmentsrpytorch_lightning.trainerr$pytorch_lightning.utilities.argparserrr rPr r rr r rrMrrJr r!rOr`rmrrrrrrrrs0        ,F