o
    ॵi                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlZd dlZd dlm	Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d	d
lmZ ejejdG dd deZdS )    N)OrderedDict)distributed)Hooks)HOOKS)LogKeysModeKeys)EnhancedEncoder)	is_master   )
LoggerHook)module_namec                       sf   e Zd ZdZ							d fdd		Z fd
dZdd Zdd Zdd ZdddZ	dd Z
  ZS )TextLoggerHooka`  Logger hook in text, Output log to both console and local json file.

    Args:
        by_epoch (bool, optional): Whether EpochBasedTrainer is used.
            Default: True.
        interval (int, optional): Logging interval (every k iterations).
            It is interval of iterations even by_epoch is true. Default: 10.
        ignore_last (bool, optional): Ignore the log of last iterations in each
            epoch if less than :attr:`interval`. Default: True.
        reset_flag (bool, optional): Whether to clear the output buffer after
            logging. Default: False.
        out_dir (str): The directory to save log. If is None, use `trainer.work_dir`
        ignore_rounding_keys (`Union[str, List]`): The keys to ignore float rounding, default 'lr'
        rounding_digits (`int`): The digits of rounding, exceeding parts will be ignored.
    T
   FNlr   c                    sV   t t| |||| || _d| _|| _g | _t|ts |d u r#|g}|| _	|| _
d S )Nr   )superr   __init__by_epochtime_sec_totout_dir_logged_keys
isinstancestrignore_rounding_keysrounding_digits)selfr   intervalignore_last
reset_flagr   r   r   	__class__ e/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/hooks/logger/text_logger_hook.pyr   %   s   
zTextLoggerHook.__init__c                    s   t t| | | jd u r|j| _t| js t r t	| j |j
d| j |j| _t| jd|j| _t|drL|jd urN| |j d S d S d S )NzText logs will be saved to {}z{}.log.jsonmeta)r   r   
before_runr   work_dirospexistsr	   osmakedirsloggerinfoformatiter
start_iterjoin	timestampjson_log_pathhasattrr#   	_dump_log)r   trainerr   r!   r"   r$   9   s   


zTextLoggerHook.before_runc                 C   sV   t j }t jj|d}t jt|d gt j|d}|jr'tj|dtj	j
d | S )N)devicei   )dtyper5   r   )op)torchcudacurrent_devicemax_memory_allocatedtensorint_distdistreduceReduceOpMAXitem)r   r4   r5   memmem_mbr!   r!   r"   _get_max_memoryK   s   
zTextLoggerHook._get_max_memoryc              	   C   s  t j}t j}t j}t j}t j}t j}t j}	|| tj	krt
|| trDg }
||  D ]\}}|
| d| d|d q+d|
}
n
| d|| d}
| jre| d||  d||  d|j d}n| d||  d|j d}||
 d	7 }| j||||g || v r|  j|| | j 7  _| j|j| j d
  }||j|j d
  }ttjt|d}||	 d| d	7 }|| d|| dd	7 }|| d|| dd	7 }| j||g n0| jr| d||  d||  d||  d}n| d||  d||  d}| j|||g g }| D ]'\}}|| jv r$qt
|tr4|| jvr4|d}|| d|  q|d	|7 }t rS|j !| d S d S )N_z: z.3e z [z][/z]	z, r
   )secondsz.3f(z) [z.4f)"r   LREPOCHITERMODE	ITER_TIMEDATA_LOAD_TIMEETAr   TRAINr   dictitemsappendr/   r   iters_per_epoch	max_itersr   extendkeysr   r   r-   r.   r   datetime	timedeltar=   floatr   r	   r*   r+   )r   log_dictr4   lr_key	epoch_keyiter_keymode_keyiter_time_keydata_load_time_keyeta_keylr_strkvallog_strtime_sec_avgeta_seceta_str	log_itemsnamer!   r!   r"   	_log_infoU   sh   (	*zTextLoggerHook._log_infoc                 C   s   t  }| D ]\}}|| jv r|n| || j||< qt rEt| jd}tj	||t
d |d W d    d S 1 s>w   Y  d S d S )Nza+)cls
)r   rU   r   _round_floatr   r	   openr1   jsondumpr   write)r   r^   json_logrg   vfr!   r!   r"   r3      s   "zTextLoggerHook._dump_logc                    s6   t |tr fdd|D S t |trt| S |S )Nc                    s   g | ]} | qS r!   )rr   ).0rC   ndigitsr   r!   r"   
<listcomp>   s    z/TextLoggerHook._round_float.<locals>.<listcomp>)r   listr]   round)r   rU   r|   r!   r{   r"   rr      s
   


zTextLoggerHook._round_floatc                 C   s~   |j tjkr| j|ddn|j}t|j | ||d}tj	 r(| 
||tj< t|fi |jj}| || | | |S )NT)
inner_iter)modeepochr-   )r   r   rS   get_iterrW   r   	get_epochr8   r9   is_availablerF   r   MEMORYrT   
log_bufferoutputro   r3   )r   r4   cur_iterr^   r!   r!   r"   log   s   

zTextLoggerHook.log)Tr   TFNr   r   )r   )__name__
__module____qualname____doc__r   r$   rF   ro   r3   rr   r   __classcell__r!   r!   r   r"   r      s     
?
r   )r[   r(   os.pathpathr&   collectionsr   rt   r8   r   r?   modelscope.metainfor   !modelscope.trainers.hooks.builderr   modelscope.utils.constantr   r   modelscope.utils.json_utilsr   modelscope.utils.torch_utilsr	   baser   register_moduler   r!   r!   r!   r"   <module>   s   