o
    8wi@2                     @   s$  d dl Z d dlZd dlmZ d dlmZmZmZmZ d dl	m
Z
 d dlZd dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZm Z  e !e"Z#dddedededef
ddZ$ddde%ddfddZ&d6ddZ'd6ddZ(d6ddZ)ddddd e*ded!ed" dedefd#d$Z+ddd e*dededef
d%d&Z,dd'ddd e*ded(ee- deddfd)d*Z.ddde/e*e/f fd+d,Z0ddd-e/e*ef ddfd.d/Z1ddd-e/e*ef ddfd0d1Z2ddd-e/e*ef ddfd2d3Z3ddd e*dededef
d4d5Z4dS )7    N)deepcopy)AnyCallableOptionalUnion)Version)_DeviceDtypeModuleMixin)
CheckpointEarlyStopping)WandbLogger)_SubprocessScriptLauncher)_get_sigkill_signal)TrainerStatus)_TunerExitException)is_overridden)rank_zero_inforank_zero_warntrainer
pl.Trainer
trainer_fnargskwargsreturnc              
   O   s  z| j jdur| j jj|g|R d| i|W S ||i |W S  ty:   t|  |   tj| j_	d| j_
Y dS  tyt } z/td ttjtj t| | |   | j j}t|tre|t  td W Y d}~dS d}~w ty } zt| | |   d| j_
 d}~ww )am  Error handling, intended to be used only for main trainer function entry points (fit, validate, test, predict)
    as all errors should funnel through them.

    Args:
        trainer_fn: one of (fit, validate, test, predict)
        *args: positional arguments to be passed to the `trainer_fn`
        **kwargs: keyword arguments to be passed to `trainer_fn`

    Nr   z=
Detected KeyboardInterrupt, attempting graceful shutdown ...   )strategylauncherlaunchr   _call_teardown_hook	_teardownr   FINISHEDstatestatusstageKeyboardInterruptr   signalSIGINTSIG_IGN
_interrupt
isinstancer   killr   exitBaseException)r   r   r   r   	exceptionr    r-   [/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py_call_and_handle_interrupt#   s4   
"



r/   r,   c                 C   sR   t j| j_t| d| | jd urt| d| | j| | j	D ]}|
d qd S )Non_exceptionfailed)r   INTERRUPTEDr    r!   _call_callback_hooks
datamodule_call_lightning_datamodule_hookr   r0   loggersfinalize)r   r,   loggerr-   r-   r.   r'   K   s   


r'   c                 C   s   | j jd usJ | j j}| j D ]}t|tr| jj|_qt	| j
dd d}|D ]
}t|dr3|j}q)| jd | jd urFt| d|d t| d|d t| d|d | jd d S )	Nc                 S   s   t | t S )N)r(   r   )r8   r-   r-   r.   <lambda>a   s    z"_call_setup_hook.<locals>.<lambda>)key
experiment	pre_setupsetupr"   
post_setup)r    fnlightning_modulemodulesr(   r   r   root_device_devicesortedr6   hasattrr;   barrierr4   r5   r3   _call_lightning_module_hook)r   r@   moduler6   r8   _r-   r-   r.   _call_setup_hookU   s"   



rK   c              
   C   s   t d| jr | j  t| d W d    n1 sw   Y  t d| jru| j @ | j " | j  t| d W d    n1 sGw   Y  W d    n1 sVw   Y  W d    d S W d    d S 1 snw   Y  d S d S )Nconfigure_sharded_modelconfigure_model)r   rA   r   model_sharded_contextrH   tensor_init_contextprecision_pluginmodule_init_context)r   r-   r-   r.   _call_configure_modelr   s    P rR   c                 C   s   | j jd usJ | j j}| jd urt| d|d t| d|d t| d|d d | j_d | j_| j	D ]}|
d q1| j  d S )Nteardownr>   success)r    r@   r4   r5   r3   rH   rA   _current_fx_name_metric_attributesr6   r7   profilerdescribe)r   r@   r8   r-   r-   r.   r      s   

r   )	pl_module	hook_namerY   zpl.LightningModulec                O   s   t | jj d|  |p| j}|d u rtdt||}t|s$d S |j}||_| j	
d|jj d|  ||i |}W d    n1 sJw   Y  ||_|S )Nz!: calling lightning module hook: z3No `LightningModule` is available to call hooks on.z[LightningModule].)logdebug	__class____name__rA   	TypeErrorgetattrcallablerU   rW   profile)r   rZ   rY   r   r   r@   prev_fx_nameoutputr-   r-   r.   rH      s   

rH   c                 O   s   t | jj d|  | jd u rtdt| j|}t|rE| j	d| jjj d|  ||i |W  d    S 1 s@w   Y  d S )Nz%: calling lightning datamodule hook: z7No `LightningDataModule` is available to call hooks on.z[LightningDataModule]r[   )
r\   r]   r^   r_   r4   r`   ra   rb   rW   rc   )r   rZ   r   r   r@   r-   r-   r.   r5      s   
  r5   )monitoring_callbacksrf   c          
   	   O   s   t | jj d|  | j}|r|j}||_| j}|du r&dd |D }n|du r1dd |D }|D ]5}t||}	t|	rh| j	
d|j d|  |	| | jg|R i | W d    n1 scw   Y  q3|rp||_d S d S )	Nz: calling callback hook: Tc                 S   s   g | ]}t |ttfr|qS r-   r(   r
   r	   .0cbr-   r-   r.   
<listcomp>       z(_call_callback_hooks.<locals>.<listcomp>Fc                 S   s   g | ]}t |ttfs|qS r-   rg   rh   r-   r-   r.   rk      rl   
[Callback]r[   )r\   r]   r^   r_   rA   rU   	callbacksra   rb   rW   rc   	state_key)
r   rZ   rf   r   r   rY   rd   rn   callbackr@   r-   r-   r.   r3      s(   

r3   c                 C   s*   i }| j D ]}| }|r|||j< q|S )zzCalled when saving a model checkpoint, calls and returns every callback's `state_dict`, keyed by
    `Callback.state_key`.)rn   
state_dictro   )r   callback_state_dictsrp   rq   r-   r-   r.   _call_callbacks_state_dict   s   

rs   
checkpointc              	   C   sz   | j }|r|j}d|_| jD ]%}| jd|j d || | j | W d   n1 s.w   Y  q|r;||_dS dS )zXCalled when saving a model checkpoint, calls every callback's `on_save_checkpoint` hook.on_save_checkpointrm   z.on_save_checkpointN)rA   rU   rn   rW   rc   ro   ru   )r   rt   rY   rd   rp   r-   r-   r.   "_call_callbacks_on_save_checkpoint   s   

rv   c              	      s   | j }|r|j}d|_|d}|du rdS t|d tdk   fdd| jD }| | }|r<tdt| d	 | jD ]%}| j	d
|j
 d || | j | W d   n1 s_w   Y  q?|rl||_dS dS )zCalled when loading a model checkpoint.

    Calls every callback's `on_load_checkpoint` hook. We have a dedicated function for this rather than using
    `_call_callback_hooks` because we have special logic for getting callback_states.

    on_load_checkpointrn   Nzpytorch-lightning_versionz1.5.0devc                    s   h | ]
} r	|j n|jqS r-   )_legacy_state_keyro   rh   is_legacy_ckptr-   r.   	<setcomp>  s    z5_call_callbacks_on_load_checkpoint.<locals>.<setcomp>zBe aware that when using `ckpt_path`, callbacks used to create the checkpoint need to be provided during `Trainer` instantiation. Please add the following callbacks: r[   rm   z.on_load_checkpoint)rA   rU   getr   rn   keysr   listrW   rc   ro   rw   )r   rt   rY   rd   callback_statescurrent_callbacks_keys
differencerp   r-   ry   r.   "_call_callbacks_on_load_checkpoint  s0   


r   c                 C   sR   | d}|du rdS | jD ]}| |j| |j}|r&t|}|| qdS )zQCalled when loading a model checkpoint, calls every callback's `load_state_dict`.rn   N)r|   rn   ro   rx   r   load_state_dict)r   rt   r   rp   r    r-   r-   r.   _call_callbacks_load_state_dict)  s   


r   c                 O   s   t | jj d|  | j}|j}||_t| j|}t|s!d S | j	
d| jjj d|  ||i |}W d    n1 sBw   Y  ||_|S )Nz: calling strategy hook: z
[Strategy]r[   )r\   r]   r^   r_   rA   rU   ra   r   rb   rW   rc   )r   rZ   r   r   rY   rd   r@   re   r-   r-   r.   _call_strategy_hook7  s    r   )r   r   r   N)5loggingr$   copyr   typingr   r   r   r   packaging.versionr   pytorch_lightningpl-lightning_fabric.utilities.device_dtype_mixinr   pytorch_lightning.callbacksr	   r
   pytorch_lightning.loggersr   &pytorch_lightning.strategies.launchersr   5pytorch_lightning.trainer.connectors.signal_connectorr    pytorch_lightning.trainer.statesr   &pytorch_lightning.utilities.exceptionsr   )pytorch_lightning.utilities.model_helpersr   %pytorch_lightning.utilities.rank_zeror   r   	getLoggerr_   r\   r/   r+   r'   rK   rR   r   strrH   r5   boolr3   dictrs   rv   r   r   r   r-   r-   r-   r.   <module>   s   
(






 $