o
    zi0                     @   s   d dl Z d dlZd dlmZ d dlmZmZmZmZm	Z	m
Z
 d dlmZ d dlZd dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZm Z  e !e"Z#dddedededef
ddZ$ddde%ddfddZ&d5ddZ'd5ddZ(d5ddZ)ddddde*ded ed! dedefd"d#Z+ddde*dededef
d$d%Z,dd&ddde*ded'ee- deddfd(d)Z.dddee*e/f fd*d+Z0ddd,ee*ef ddfd-d.Z1ddd,ee*ef ddfd/d0Z2ddd,ee*ef ddfd1d2Z3ddde*dededef
d3d4Z4dS )6    N)deepcopy)AnyCallableDictOptionalTypeUnion)Version)_DeviceDtypeModuleMixin)
CheckpointEarlyStopping)_SubprocessScriptLauncher)_get_sigkill_signal)TrainerStatus)_TunerExitException)is_overridden)rank_zero_inforank_zero_warntrainer
pl.Trainer
trainer_fnargskwargsreturnc              
   O   s  z| j jdur| j jj|g|R d| i|W S ||i |W S  ty:   t|  |   tj| j_	d| j_
Y dS  tyt } z/td ttjtj t| | |   | j j}t|tre|t  td W Y d}~dS d}~w ty } zt| | |   d| j_
 d}~ww )am  Error handling, intended to be used only for main trainer function entry points (fit, validate, test, predict)
    as all errors should funnel through them.

    Args:
        trainer_fn: one of (fit, validate, test, predict)
        *args: positional arguments to be passed to the `trainer_fn`
        **kwargs: keyword arguments to be passed to `trainer_fn`

    Nr   z=
Detected KeyboardInterrupt, attempting graceful shutdown ...   )strategylauncherlaunchr   _call_teardown_hook	_teardownr   FINISHEDstatestatusstageKeyboardInterruptr   signalSIGINTSIG_IGN
_interrupt
isinstancer   killr   exitBaseException)r   r   r   r   	exceptionr    r.   R/home/ubuntu/.local/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py_call_and_handle_interrupt"   s4   
"



r0   r-   c                 C   sR   t j| j_t| d| | jd urt| d| | j| | j	D ]}|
d qd S )Non_exceptionfailed)r   INTERRUPTEDr!   r"   _call_callback_hooks
datamodule_call_lightning_datamodule_hookr   r1   loggersfinalize)r   r-   loggerr.   r.   r/   r(   J   s   


r(   c                 C   s   | j jd usJ | j j}| j D ]}t|tr| jj|_q| j	D ]
}t
|dr+|j}q!| jd | jd ur>t| d|d t| d|d t| d|d | jd d S )N
experiment	pre_setupsetupr#   
post_setup)r!   fnlightning_modulemodulesr)   r
   r   root_device_devicer7   hasattrr:   barrierr5   r6   r4   _call_lightning_module_hook)r   r?   moduler9   _r.   r.   r/   _call_setup_hookT   s    




rI   c              
   C   s   t d| jr | j  t| d W d    n1 sw   Y  t d| jru| j @ | j " | j  t| d W d    n1 sGw   Y  W d    n1 sVw   Y  W d    d S W d    d S 1 snw   Y  d S d S )Nconfigure_sharded_modelconfigure_model)r   r@   r   model_sharded_contextrF   tensor_init_contextprecision_pluginmodule_init_context)r   r.   r.   r/   _call_configure_modelm   s   $P rP   c                 C   s   | j jd usJ | j j}| jd urt| d|d t| d|d t| d|d d | j_d | j_| j	D ]}|
d q1| j  d S )Nteardownr=   success)r!   r?   r5   r6   r4   rF   r@   _current_fx_name_metric_attributesr7   r8   profilerdescribe)r   r?   r9   r.   r.   r/   r   z   s   

r   )	pl_module	hook_namerW   zpl.LightningModulec                O   s   t | jj d|  |p| j}|d u rtdt||}t|s$d S |j}||_| j	
d|jj d|  ||i |}W d    n1 sJw   Y  ||_|S )Nz!: calling lightning module hook: z3No `LightningModule` is available to call hooks on.z[LightningModule].)logdebug	__class____name__r@   	TypeErrorgetattrcallablerS   rU   profile)r   rX   rW   r   r   r?   prev_fx_nameoutputr.   r.   r/   rF      s   

rF   c                 O   s   t | jj d|  | jd u rtdt| j|}t|rE| j	d| jjj d|  ||i |W  d    S 1 s@w   Y  d S )Nz%: calling lightning datamodule hook: z7No `LightningDataModule` is available to call hooks on.z[LightningDataModule]rY   )
rZ   r[   r\   r]   r5   r^   r_   r`   rU   ra   )r   rX   r   r   r?   r.   r.   r/   r6      s   
  r6   )monitoring_callbacksrd   c          
   	   O   s   t | jj d|  | j}|r|j}||_| j}|du r&dd |D }n|du r1dd |D }|D ]5}t||}	t|	rh| j	
d|j d|  |	| | jg|R i | W d    n1 scw   Y  q3|rp||_d S d S )	Nz: calling callback hook: Tc                 S   s   g | ]}t |ttfr|qS r.   r)   r   r   .0cbr.   r.   r/   
<listcomp>       z(_call_callback_hooks.<locals>.<listcomp>Fc                 S   s   g | ]}t |ttfs|qS r.   re   rf   r.   r.   r/   ri      rj   
[Callback]rY   )rZ   r[   r\   r]   r@   rS   	callbacksr_   r`   rU   ra   	state_key)
r   rX   rd   r   r   rW   rb   rl   callbackr?   r.   r.   r/   r4      s(   

r4   c                 C   s*   i }| j D ]}| }|r|||j< q|S )zzCalled when saving a model checkpoint, calls and returns every callback's `state_dict`, keyed by
    `Callback.state_key`.)rl   
state_dictrm   )r   callback_state_dictsrn   ro   r.   r.   r/   _call_callbacks_state_dict   s   

rq   
checkpointc              	   C   sz   | j }|r|j}d|_| jD ]%}| jd|j d || | j | W d   n1 s.w   Y  q|r;||_dS dS )zXCalled when saving a model checkpoint, calls every callback's `on_save_checkpoint` hook.on_save_checkpointrk   z.on_save_checkpointN)r@   rS   rl   rU   ra   rm   rs   )r   rr   rW   rb   rn   r.   r.   r/   "_call_callbacks_on_save_checkpoint   s   

rt   c              	      s   | j }|r|j}d|_|d}|du rdS t|d tdk   fdd| jD }| | }|r<tdt| d	 | jD ]%}| j	d
|j
 d || | j | W d   n1 s_w   Y  q?|rl||_dS dS )zCalled when loading a model checkpoint.

    Calls every callback's `on_load_checkpoint` hook. We have a dedicated function for this rather than using
    `_call_callback_hooks` because we have special logic for getting callback_states.

    on_load_checkpointrl   Nzpytorch-lightning_versionz1.5.0devc                    s   h | ]
} r	|j n|jqS r.   )_legacy_state_keyrm   rf   is_legacy_ckptr.   r/   	<setcomp>  s    z5_call_callbacks_on_load_checkpoint.<locals>.<setcomp>zBe aware that when using `ckpt_path`, callbacks used to create the checkpoint need to be provided during `Trainer` instantiation. Please add the following callbacks: rY   rk   z.on_load_checkpoint)r@   rS   getr	   rl   keysr   listrU   ra   rm   ru   )r   rr   rW   rb   callback_statescurrent_callbacks_keys
differencern   r.   rw   r/   "_call_callbacks_on_load_checkpoint   s0   


r   c                 C   sR   | d}|du rdS | jD ]}| |j| |j}|r&t|}|| qdS )zQCalled when loading a model checkpoint, calls every callback's `load_state_dict`.rl   N)rz   rl   rm   rv   r   load_state_dict)r   rr   r}   rn   r!   r.   r.   r/   _call_callbacks_load_state_dict   s   


r   c                 O   s   t | jj d|  | j}|j}||_t| j|}t|s!d S | j	
d| jjj d|  ||i |}W d    n1 sBw   Y  ||_|S )Nz: calling strategy hook: z
[Strategy]rY   )rZ   r[   r\   r]   r@   rS   r_   r   r`   rU   ra   )r   rX   r   r   rW   rb   r?   rc   r.   r.   r/   _call_strategy_hook.  s    r   )r   r   r   N)5loggingr%   copyr   typingr   r   r   r   r   r   packaging.versionr	   pytorch_lightningpl-lightning_fabric.utilities.device_dtype_mixinr
   pytorch_lightning.callbacksr   r   &pytorch_lightning.strategies.launchersr   5pytorch_lightning.trainer.connectors.signal_connectorr    pytorch_lightning.trainer.statesr   &pytorch_lightning.utilities.exceptionsr   )pytorch_lightning.utilities.model_helpersr   %pytorch_lightning.utilities.rank_zeror   r   	getLoggerr]   rZ   r0   r,   r(   rI   rP   r   strrF   r6   boolr4   dictrq   rt   r   r   r   r.   r.   r.   r/   <module>   s    
(






 $