o
    }oiZ                     @   s\   d dl mZ d dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZ G dd deZd	S )
    )defaultdict)Any)ProgressBar)STEP_OUTPUT)get_num_microbatches)overridec                       s  e Zd ZdZddgdgfdedee dee f fdd	Zd
d Zdd Z	dd Z
edefddZedd Zedd Zedd Zedd Zedd Zejdd Zededdfd d!Zededdfd"d#Zed$d% Zed&d' Ze	(d=d)d*d+d,d-ed.ed/eddfd0d1Ze	(d=d)d*d+d,d2ed-ed.ed/eddfd3d4Ze	(d=d)d*d+d,d-ed.ed/eddfd5d6Ze	(d=d)d*d+d,d2ed-ed.ed/eddfd7d8Zd9d: Zd;d< Z   Z!S )>ProgressPrintera   
    Callback for logging progress in Megatron. Prints status in terms of global batches rather than microbatches.
    Recommended over MegatronProgressBar for non-interactive settings

    Args:
        log_interval (int): determines how frequently (in steps) to print the progress.
        skip_accumulate_metrics (list[str]): for all metrics in this list, value logged will
            simply reflect the latest value rather than averaging over the log interval.
        exclude_metrics (list[str]): any metrics to exclude from logging.
       global_stepv_numlog_intervalskip_accumulate_metricsexclude_metricsc                    sN   d| _ d| _d| _t|| _|| _|| _tdd | _|dk| _	t
   d S )NTraining
ValidationTestingc                   S      dS Ng         r   r   r   e/home/ubuntu/.local/lib/python3.10/site-packages/nemo/lightning/pytorch/callbacks/progress_printer.py<lambda>2       z*ProgressPrinter.__init__.<locals>.<lambda>r   )_train_description_validation_description_test_descriptionint_log_intervalr   r   r   total_metrics_dict_is_disabledsuper__init__)selfr   r   r   	__class__r   r   r    $   s   

zProgressPrinter.__init__c                 C   s`   |}|  D ]'\}}t|tr"| r"t|}|d| d| 7 }q|d| d|d7 }q|S )Nz | z: z.4)items
isinstancefloat
is_integerr   )r!   prefixmetrics
log_stringmetricvalr   r   r   format_string7   s   zProgressPrinter.format_stringc                 C   
   d| _ d S )NTr   r!   r   r   r   disableA      
zProgressPrinter.disablec                 C   r.   )NFr/   r0   r   r   r   enableD   r2   zProgressPrinter.enablereturnc                 C      | j S Nr/   r0   r   r   r   is_disabledG      zProgressPrinter.is_disabledc                 C   sV   i }| j D ]#}|| jv st| j | ttfs| j | ||< q| j | | j ||< q|S r6   )r   r   r%   r   r&   r   )r!   average_dictkeyr   r   r   average_metrics_dictK   s   
z$ProgressPrinter.average_metrics_dictc                 C   r5   r6   )r   r0   r   r   r   train_descriptionU   r8   z!ProgressPrinter.train_descriptionc                 C   r5   r6   r   r0   r   r   r   validation_descriptionY   r8   z&ProgressPrinter.validation_descriptionc                 C   r5   r6   )r   r0   r   r   r   test_description]   r8   z ProgressPrinter.test_descriptionc                 C   r5   r6   r   r0   r   r   r   r   a   r8   zProgressPrinter.log_intervalc                 C   s
   || _ d S r6   r@   )r!   r,   r   r   r   r   e      
_Nc                 G   s   d| j  | _d S )NzSanity checking )r>   r   r!   rB   r   r   r   on_sanity_check_starti   s   z%ProgressPrinter.on_sanity_check_startc                 G   r.   )Nr   r=   rC   r   r   r   on_sanity_check_endm   rA   z#ProgressPrinter.on_sanity_check_endc                 G   s"   |j dkr|j | _d S |j| _d S Nr   )	max_stepstotalnum_training_batches)r!   trainerrB   r   r   r   on_train_startq   s   
zProgressPrinter.on_train_startc                 O   s  |j j}| jr	d S | ||}|D ])}|| jv rq|| jv s't|| ttfs/|| | j	|< q| j	|  || 7  < q| 
|r| jd|j d|d  d| jd   }| || j}	t|	 t|j dd rw|j j}
| |
}|rwt|dd tdd	 | _	d S d S )
Nz epoch z, iteration r	   /timersT)flushc                   S   r   r   r   r   r   r   r   r      r   z4ProgressPrinter.on_train_batch_end.<locals>.<lambda>)strategycurrent_epoch_stepr7   get_metricsr   r   r%   r   r&   r   
should_logr<   current_epochrH   r-   r;   printgetattrrM   log_megatron_timersr   )r!   rJ   	pl_modulerB   __nr)   r:   r(   r*   rM   megatron_log_stringr   r   r   on_train_batch_end{   s*   

(
z"ProgressPrinter.on_train_batch_endr   rJ   z
pl.TrainerrW   zpl.LightningModulebatch	batch_idxdataloader_idxc                 C   sD   |  |sd S t| jtdkrtd| _d S t| jt  | _d S )Ninf)has_dataloader_changedr&   $total_val_batches_current_dataloadertotal_validation_stepsr   r   r!   rJ   rW   r\   r]   r^   r   r   r   on_validation_batch_start   s
   
	z)ProgressPrinter.on_validation_batch_startoutputsc                 C   sJ   | j rd S |d t  }| |r#t| jdt| d| j   d S d S Nr	   z: iteration rL   )r7   r   rR   rT   r>   r   rb   r!   rJ   rW   re   r\   r]   r^   rY   r   r   r   on_validation_batch_end   s   

$z'ProgressPrinter.on_validation_batch_endc                 C   s$   |  |sd S t| jt  | _d S r6   )r`   r   %total_test_batches_current_dataloaderr   total_test_stepsrc   r   r   r   on_test_batch_start   s   
	z#ProgressPrinter.on_test_batch_startc                 C   sJ   | j rd S t|d t  }| |r#t| jd| d| j   d S d S rf   )r7   r   r   rR   rT   r?   rb   rg   r   r   r   on_test_batch_end   s   

 z!ProgressPrinter.on_test_batch_endc                 C   s   || j  dkS rF   )r   )r!   rY   r   r   r   rR      s   zProgressPrinter.should_logc                 C   s$   |j d | jd}|d ur|d S d S )N)names
normalizer
)get_all_timers_stringr   )r!   rM   output_stringr   r   r   rV      s   z#ProgressPrinter.log_megatron_timers)r   )"__name__
__module____qualname____doc__r   liststrr    r-   r1   r3   propertyboolr7   r;   r<   r>   r?   r   setterr   r   rD   rE   rK   r[   rd   r   rh   rk   rl   rR   rV   __classcell__r   r   r"   r   r      s    

	





	
r   N)collectionsr   typingr   $lightning.pytorch.callbacks.progressr   !lightning.pytorch.utilities.typesr   )megatron.core.num_microbatches_calculatorr   typing_extensionsr   r   r   r   r   r   <module>   s   