o
    }oi[                     @   s   d dl Z d dlmZ d dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZmZmZ G d	d
 d
e	ZdejddfddZG dd dejeZdS )    Ndeepcopy)_TrainingEpochLoop)_DataFetcher)Self)	to_fabric)Fabric)IOMixinserializationtrack_ioc                       sX   e Zd ZdZdef fddZddededdf fd	d
Zde	ddf fddZ
  ZS )NoValOnRestartTrainingLoopz
    Extend the PTL Epoch loop to skip validation when restarting.
    This happens when resuming a checkpoint that has already run validation, but loading restores
    the training state before validation has run.
    returnc                    s   | j rdS t |S NF)skip_val_on_restartsuper_should_check_val_fxselfdata_fetcher	__class__ R/home/ubuntu/.local/lib/python3.10/site-packages/nemo/lightning/pytorch/trainer.pyr   $   s   z/NoValOnRestartTrainingLoop._should_check_val_fx 
state_dictprefixNc                    s   t  || d| _d S )NT)r   load_state_dictr   )r   r   r   r   r   r   r   )   s   
z*NoValOnRestartTrainingLoop.load_state_dictr   c                    s   t  | d| _d S r   )r   advancer   r   r   r   r   r   .   s   
z"NoValOnRestartTrainingLoop.advance)r   )__name__
__module____qualname____doc__boolr   dictstrr   r   r   __classcell__r   r   r   r   r      s
    r   trainerr   c                 C   s:   t | jjtstdt d S t| | j| j	}|| j_d S )NzFDetected custom epoch loop. Skipping no validation on restart support.)

isinstancefit_loop
epoch_loopr   warningswarnUserWarningr   	min_steps	max_steps)r&   loopr   r   r   -configure_no_restart_validation_training_loop4   s
   r0   c                   @   s8   e Zd Zdd Zdeje fddZd	defddZ	dS )
Trainerc                 C   sV   t |ttfrt |tr| }|D ]}| | qdS tt|s)tt| dS )zXRecurse to the leaves of a container and add io functionality to non-serializable leavesN)	r'   r#   listvaluesadd_ior
   find_node_traversertyper   )r   objitemr   r   r   r4   ?   s   
zTrainer.add_ior   c                 K   s2   dd |  D }| | tjt| fi |S )Nc                 S   s   i | ]	\}}|t |qS r   r   ).0kvr   r   r   
<dictcomp>M   s    z#Trainer.io_init.<locals>.<dictcomp>)itemsr4   fdlConfigr6   )r   kwargs
cfg_kwargsr   r   r   io_initK   s   
zTrainer.io_initNc           	   	   C   s   d\}}}}}t | jdr| jj}t | jdr| jj}t | jdr4| jj}t|tjr0t|}t	|}t | jdrM| jj
}t|tjrIt|}t	|}t | jdrW| jj}t|||||||d}|S )N)NNNNNdevicesacceleratorstrategyplugins	num_nodes)rC   rD   rE   rF   	callbacksloggersrG   )hasattr__io__rC   rD   rE   r'   r>   r?   buildr   rF   rG   r   )	r   rH   rI   rD   rC   rE   rF   rG   outr   r   r   r   R   s6   


zTrainer.to_fabric)NN)
r   r   r    r4   r>   r?   r   rB   r   r   r   r   r   r   r1   >   s    r1   )r*   copyr   fiddler>   lightning.pytorchpytorchpllightning.pytorch.loopsr    lightning.pytorch.loops.fetchersr   typing_extensionsr    nemo.lightning.fabric.conversionr   nemo.lightning.fabric.fabricr   nemo.lightning.io.mixinr	   r
   r   r   r1   r0   r   r   r   r   <module>   s   
