o
    }oi
                     @   s:   d dl mZ d dlmZ G dd deZdefddZdS )	    )TQDMProgressBar)	_update_nc                       s0   e Zd ZdZ fddZdd Zdd Z  ZS )MegatronProgressBarz
    Add MegatronProgressBar to remove 's/it' and display progress per step instead of per microbatch
    for megatron models.
    c                    s   t   | _d| j_| jS )z9
        Override bar_format to not have 's/it'.
        zV{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}{postfix}])superinit_train_tqdmbar
bar_format)self	__class__ a/home/ubuntu/.local/lib/python3.10/site-packages/nemo/lightning/pytorch/callbacks/progress_bar.pyr      s   z#MegatronProgressBar.init_train_tqdmc                 G   sD   |j dkr	|j }n|j}| j| d| j_| jd|j  d S )Nr   zEpoch )	max_stepsnum_training_batchestrain_progress_barresetinitialset_descriptioncurrent_epoch)r	   trainer_r   r   r   r   on_train_epoch_start!   s   
z(MegatronProgressBar.on_train_epoch_startc                 O   sD   |j j}| || jjr t| j| | jj| ||dd dS dS )z}
        Override parent class on_train_batch_end to update progress bar per global batch instead of per microbatch.
        F)refreshN)strategycurrent_epoch_step_should_updater   totalr   set_postfixget_metrics)r	   r   	pl_moduler   __nr   r   r   on_train_batch_end-   s
   z&MegatronProgressBar.on_train_batch_end)__name__
__module____qualname____doc__r   r   r"   __classcell__r   r   r
   r   r      s
    r   returnc                  C   s8   ddl m}  |  }|j}|j}|j}|||  }|| S )Nr   )AppState)
nemo.utilsr)   pipeline_model_parallel_sizetensor_model_parallel_size
world_size)r)   	app_stater+   r,   r-   data_parallel_group_lenr   r   r   calculate_data_parallel_groups7   s   r0   N)$lightning.pytorch.callbacks.progressr   2lightning.pytorch.callbacks.progress.tqdm_progressr   r   intr0   r   r   r   r   <module>   s   $