o
    iL                     @   sh  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZmZmZmZmZmZmZ ddlZddlZddlZddlmZ ddlmZmZ eeee ej!ej"f Z#d	d
hZ$d"de#de#ddfddZ%ded de#fddZ&de'fddZ(G dd dZ)ej*ddG dd de)Z+ej*ddG dd de)Z,G dd dZ-G d d! d!Z.dS )#zReporter module.    N)defaultdict)contextmanager)Path)ContextManagerDictListOptionalSequenceTupleUnion)parse)check_argument_typescheck_return_typetimetotal_countvweightreturnReportedValuec                 C   s   t  sJ t| tjtjfr$t| jdkr tdt	| j | 
 } t|tjtjfrCt|jdkr?tdt	|j |
 }|d urMt| |}nt| }t|sWJ |S )N   zv must be 0 or 1 dimension: z!weight must be 0 or 1 dimension: )r   
isinstancetorchTensornpndarrayprodshape
ValueErrorlenitemWeightedAverageAverager   )r   r   retval r#   J/home/ubuntu/.local/lib/python3.10/site-packages/espnet2/train/reporter.pyto_reported_value   s   
r%   valuesc                    sz  t  sJ | D ]}t|t| d s"tdt| dt| d  qt| dkr2td tj}nt| d t	rDt
dd | D }nqt| d trt  t| D ]\}}t|jrbt|jsg | qR fddt| D } t| dkrtdd	 t| D }td
d	 t| D }|dkrtd tj}n|| }ntd tj}ntdt| d  t|sJ |S )Nr   z,Can't use different Reported type together:  != zNo stats foundc                 S   s   g | ]}|j qS r#   )value).0r   r#   r#   r$   
<listcomp>;   s    zaggregate.<locals>.<listcomp>c                    s   g | ]
\}}| vr|qS r#   r#   r)   ir   invalid_indicesr#   r$   r*   C   s    c                 s   s    | ]\}}|j V  qd S N)r   r+   r#   r#   r$   	<genexpr>G   s    zaggregate.<locals>.<genexpr>c                 s   s     | ]\}}|j |j V  qd S r/   )r(   r   r+   r#   r#   r$   r0   H   s    zweight is zerozNo valid stats foundtype=)r   r   typer   r   warningswarnr   nanr!   nanmeanr    set	enumerateisfiniter(   r   addsumNotImplementedErrorr   )r&   r   r"   r,   sum_weights	sum_valuer#   r-   r$   	aggregate,   sF   






r?   keyc                 C   s.   |  drdS |  drdS |  drdS dS )Nvalidzvalid/trainztrain/attnzattn/zmetrics/)
startswithr@   r#   r#   r$   wandb_get_prefixX   s   


rF   c                   @   s   e Zd ZdS )r   N)__name__
__module____qualname__r#   r#   r#   r$   r   b   s    T)frozenc                   @   s   e Zd ZU eed< dS )r!   r(   N)rG   rH   rI   Num__annotations__r#   r#   r#   r$   r!   f   s   
 r!   c                   @   s&   e Zd ZU eeef ed< eed< dS )r    r(   r   N)rG   rH   rI   r
   rK   rL   r#   r#   r#   r$   r    k   s   
 r    c                
   @   s   e Zd ZdZdededefddZdefdd	Zdefd
dZdd Z		d"de
eeeee
eef f  f deddfddZd#dededefddZd"defddZd"defddZd$ddZedefddZdefd d!ZdS )%SubReporterzVThis class is used in Reporter.

    See the docstring of Reporter for the usage.
    r@   epochr   c                 C   sH   t  sJ || _|| _t | _tt| _d| _	|| _
d| _t | _d S )NFr   )r   r@   rN   r   perf_counter
start_timer   liststats	_finishedr   countr7   _seen_keys_in_the_step)selfr@   rN   r   r#   r#   r$   __init__w   s   


zSubReporter.__init__r   c                 C      | j S )z1Returns the number of iterations over all epochs.)r   rV   r#   r#   r$   get_total_count   s   zSubReporter.get_total_countc                 C   rX   r/   rN   rY   r#   r#   r$   	get_epoch      zSubReporter.get_epochc                 C   s   | j  D ]E\}}|| jvr:t|d tr|ttjd nt|d t	r/|ttj nt
dt|d  t|| jksJJ t|| jfqt | _dS )z4Close up this step and reset state for the next stepr   r1   N)rR   itemsrU   r   r    appendr%   r   r5   r!   r<   r2   r   rT   r7   )rV   r@   
stats_listr#   r#   r$   next   s   
"zSubReporter.nextNrR   r   c                    s  t  sJ jrtdtjdkr! jd7  _ jd7  _| D ][\}}|tv r4t| d|jv r@t| d|d u rGt	j
}t|||jvrrtt	j
|d u rZd nd j|  fddtjD  nj|  j| q%d S )NAlready finishedr   r   z is reserved.z is registered twice.c                 3   s&    | ]}|j d  krn V  qdS )r   N)rT   )r)   r,   r5   rrV   r#   r$   r0      s    
z'SubReporter.register.<locals>.<genexpr>)r   rS   RuntimeErrorr   rU   r   rT   r^   	_reservedr   r5   r%   rR   extendranger_   r:   )rV   rR   r   key2r   r#   rc   r$   register   s,   




zSubReporter.registerstartendc           	      C   s<  | j rtd|d u rd}|dk r| j| }|d u r| j}| jdks&||kr(dS | j d| j d|d  d| d}t| j D ]Z\}\}}t|| jksWJ t|| jf||| }|dkrk|t|krk|d	7 }t	|}t
|d
kr|| d|d7 }qAt
|dkr|| d|d7 }qA|| d|d7 }qA|S )Nrb   r    zepoch::r   -zbatch: ,      @@=.3eMbP?.3f)rS   re   rT   rN   r@   r8   rR   r^   r   r?   abs)	rV   rk   rl   messageidxri   r`   r&   r   r#   r#   r$   log_message   s.   
$ zSubReporter.log_messagec                 C   s|   |d u rd}|dk r| j | }| j D ]'\}}t|| j ks(J t|| j f||d  }t|}|| || j qd S )Nr   )rT   rR   r^   r   r?   
add_scalarr   )rV   summary_writerrk   ri   r`   r&   r   r#   r#   r$   tensorboard_add_scalar   s   
 z"SubReporter.tensorboard_add_scalarc                 C   s   dd l }|d u r
d}|dk r| j| }i }| j D ]&\}}t|| jks.J t|| jf||d  }t|}||t|| < q| j|d< || d S )Nr   	iteration)	wandbrT   rR   r^   r   r?   rF   r   log)rV   rk   r~   dri   r`   r&   r   r#   r#   r$   	wandb_log   s   
 
zSubReporter.wandb_logc                 C   s
   d| _ d S )NT)rS   rY   r#   r#   r$   finished   s   
zSubReporter.finishednamec                 c   s.    t  }|V  t  | }| ||i d S r/   )r   rO   rj   )rV   r   rk   tr#   r#   r$   measure_time   s
   zSubReporter.measure_timec                 c   sX    t |}	 zt }t|}t | }| ||i |V  W n
 ty*   Y d S w qr/   )iterr   rO   ra   rj   StopIteration)rV   iterabler   iteratorrk   r"   r   r#   r#   r$   measure_iter_time  s   
zSubReporter.measure_iter_timer/   NN)r   N)rG   rH   rI   __doc__strintrW   rZ   r\   ra   r   r   r   rK   rj   ry   r|   r   r   r   r   r   r#   r#   r#   r$   rM   q   s*    
%
rM   c                   @   sD  e Zd ZdZd?defddZdefddZdedd	fd
dZed@de	dede
e fddZd@de	dedefddZdedd	fddZde	de	de	deeeef  fddZde	de	de	dee fddZde	de	de	dee fddZd?de	de	de	dedef
ddZ				dAded e	de	de	dedefd!d"Zd@de	de	dedefd#d$Zd@dede	fd%d&Zd@de	de	defd'd(Zd@dedee	d)f fd*d+Zd@de	dedee	d)f fd,d-Zd@dedeee	e	f d)f fd.d/Zd0ee	ef fd1d2Z d3e!e	 de	fd4d5Z"		dAded e	fd6d7Z#d@defd8d9Z$d:d; Z%d<e&fd=d>Z'd	S )BReportera  Reporter class.

    Examples:

        >>> reporter = Reporter()
        >>> with reporter.observe('train') as sub_reporter:
        ...     for batch in iterator:
        ...         stats = dict(loss=0.2)
        ...         sub_reporter.register(stats)

    r   rN   c                 C   s0   t  sJ |dk rtd| || _i | _d S Nr   epoch must be 0 or more: )r   r   rN   rR   rV   rN   r#   r#   r$   rW      s
   

zReporter.__init__r   c                 C   rX   r/   r[   rY   r#   r#   r$   r\   )  r]   zReporter.get_epochNc                 C   s    |dk rt d| || _d S r   )r   rN   r   r#   r#   r$   	set_epoch,  s   
zReporter.set_epochr@   c                 c   s"    |  ||}|V  | | d S r/   )start_epochfinish_epoch)rV   r@   rN   sub_reporterr#   r#   r$   observe1  s   zReporter.observec                 C   s   |d ur|dk rt d| || _| jd | jvs$|| j| jd  vr:| jd dkr7td| jd  d d}n| j| jd  | d }t|| j|}| j|d  |S )Nr   r   r   z The stats of the previous epoch=zdoesn't exist.r   )r   rN   rR   r3   r4   rM   pop)rV   r@   rN   r   r   r#   r#   r$   r   8  s   $zReporter.start_epochr   c                 C   s   | j |j krtd| j  d|j  i }|j D ]\}}t|}|||< qtjt |j	 d|d< |j
|d< ttjtdkrPtj rOtj d |d< ntj retj d	kretj d |d
< || j| j i |j< |  d S )Nz'Don't change epoch during observation: r'   )secondsr   r   z1.4.0i   @gpu_max_cached_mem_GBr   gpu_cached_mem_GB)rN   re   rR   r^   r?   datetime	timedeltar   rO   rP   r   Vr   __version__cudais_initializedmax_memory_reservedis_availablemax_memory_cached
setdefaultr@   r   )rV   r   rR   ri   r&   r   r#   r#   r$   r   O  s2   



zReporter.finish_epochri   modec                    s   |dvrt d|  st  d d   fddjD }|dkr9t|dd	 d
}|S t|dd	 d
}|S )zReturn the epoch which resulted the best value.

        Example:
            >>> val = reporter.sort_epochs_and_values('eval', 'loss', 'min')
            >>> e_1best, v_1best = val[0]
            >>> e_2best, v_2best = val[1]
        )minmaxzmode must min or max: .z is not found: c                    s"   g | ]}|j |    fqS r#   )rR   r)   er@   ri   rV   r#   r$   r*   |  s   " z3Reporter.sort_epochs_and_values.<locals>.<listcomp>r   c                 S   s   | d S Nr   r#   xr#   r#   r$   <lambda>  s    z1Reporter.sort_epochs_and_values.<locals>.<lambda>rE   c                 S   s
   | d  S r   r#   r   r#   r#   r$   r     s   
 )r   hasKeyErrorget_all_keysrR   sorted)rV   r@   ri   r   r&   r#   r   r$   sort_epochs_and_valuesl  s   
zReporter.sort_epochs_and_valuesc                 C      dd |  |||D S )Nc                 S   s   g | ]\}}|qS r#   r#   r)   r   r   r#   r#   r$   r*         z(Reporter.sort_epochs.<locals>.<listcomp>r   rV   r@   ri   r   r#   r#   r$   sort_epochs     zReporter.sort_epochsc                 C   r   )Nc                 S   s   g | ]\}}|qS r#   r#   r   r#   r#   r$   r*     r   z(Reporter.sort_values.<locals>.<listcomp>r   r   r#   r#   r$   sort_values  r   zReporter.sort_valuesnbestc                 C   s   |  |||| S r/   )r   )rV   r@   ri   r   r   r#   r#   r$   get_best_epoch  s   zReporter.get_best_epochpatiencekey1c                 C   sf   |d u rt }|d u r|  }| |||}|| |kr1|d| d| d||  d| d	 dS dS )Nz[Early stopping] r   z has not been improved z2 epochs continuously. The training was stopped at rN   TF)loggingr\   r   info)rV   r   r   ri   r   rN   logger
best_epochr#   r#   r$   check_early_stopping  s   	zReporter.check_early_stoppingc                 C   s:   |d u r|   }|| jv o|| j| v o|| j| | v S r/   )r\   rR   rV   r@   ri   rN   r#   r#   r$   r     s   
zReporter.hasc           	      C   sF  |d u r|   }d}| j|  D ]\}}d}| D ]b\}}|d ur}t|dkr-|d7 }t|tr_t|dkrC|| d|d7 }qt|dkrT|| d|d7 }q|| d|d7 }qt|tjrtt	
|}|| d| 7 }q|| d| 7 }qt|dkrt|dkr|| d	7 }n|d7 }|d
| d| 7 }q|S )Nrm   r   rp   rq   rr   rs   rt   ru   zepoch results: [z] )r\   rR   r^   r   r   floatrv   r   r   humanfriendlyformat_timespan)	rV   rN   rw   r@   r   _messageri   r   _vr#   r#   r$   ry     s6   

zReporter.log_messagec                 C   sJ   |  ||st| d| d|   |d u r|  }| j| | | S )Nr   z is not found in stats: )r   r   r   r\   rR   r   r#   r#   r$   	get_value  s
   zReporter.get_value.c                 C   s   |du r|   }t| j| S )zReturns keys1 e.g. train,eval.N)r\   tuplerR   r   r#   r#   r$   get_keys  s   zReporter.get_keysc                 C   s4   |du r|   }| j| | }tdd |D }|S )zReturns keys2 e.g. loss,acc.Nc                 s   s    | ]	}|d vr|V  qdS )r   r   Nr#   r)   kr#   r#   r$   r0     s    z%Reporter.get_keys2.<locals>.<genexpr>)r\   rR   r   )rV   r@   rN   r   keys2r#   r#   r$   	get_keys2  s
   zReporter.get_keys2c                 C   sN   |d u r|   }g }| j| D ]}| j| | D ]	}|||f qqt|S r/   )r\   rR   r_   r   )rV   rN   all_keysr@   ri   r#   r#   r$   r     s   zReporter.get_all_keys
output_dirc                    sv   t jfdd D  }|D ](  fdd D }| }|  d }|jjddd || qdS )z,Plot stats using Matplotlib and save images.c                    s   g | ]	}t  |qS r#   )r7   r   r   rY   r#   r$   r*     s    z,Reporter.matplotlib_plot.<locals>.<listcomp>c                    s   g | ]}  |v r|qS r#   )r   r   ri   rV   r#   r$   r*     s    z.pngT)parentsexist_okN)r7   unionr   _plot_statsparentmkdirsavefig)rV   r   r   keyspltpr#   r   r$   matplotlib_plot  s   zReporter.matplotlib_plotr   c                    s  t  sJ t|trtd| ddd l}|d dd lm} dd lm	} |
  td d }|D ]"  fdd|D }t|t|ksPJ d|j|| d	d
 q7|  | d |  |jdd |d | |  |S )Nz
Input as []r   aggr   c                    sN   g | ]#}|j v r" j | v r"j |   v r"j |    ntjqS r#   )rR   r   r5   r   r   r#   r$   r*     s    
z(Reporter._plot_stats.<locals>.<listcomp>zBug?r   )labelmarkerz	 vs epochT)integerrN   )r   r   r   	TypeError
matplotlibusematplotlib.pyplotpyplotmatplotlib.tickertickerclfr   aranger\   r   plotlegendtitlegca	get_xaxisset_major_locatorMaxNLocatorxlabelylabelgrid)rV   r   ri   r   r   r   epochsyr#   r   r$   r     s,   




zReporter._plot_statsc                 C   s   |d u r|   }| j| d d }|dkr|d|| |d ur&t|g}n| |}|D ]}| |D ]}|| | j| | | | q4q-d S )NrB   r   
iter_epoch)r\   rR   rz   r   r   r   )rV   r{   rN   r   r   key1_iteratorri   r#   r#   r$   r|     s   
zReporter.tensorboard_add_scalarc                 C   s   dd l }|d u r|  }i }| |D ](}| j| | D ]}|dv r#q| d| d}| j| | | |t|| < qq||d< || d S )Nr   r   __epochrN   )r~   r\   r   rR   rF   r   )rV   rN   r~   r   r   ri   r@   r#   r#   r$   r   /  s    zReporter.wandb_logc                 C   s   | j | jdS )NrR   rN   r  rY   r#   r#   r$   
state_dict?  s   zReporter.state_dictr  c                 C   s   |d | _ |d | _d S )NrN   rR   )rN   rR   )rV   r  r#   r#   r$   load_state_dictB  s   
zReporter.load_state_dict)r   r/   r   )(rG   rH   rI   r   r   rW   r\   r   r   r   r   rM   r   r   r   r   r
   r   r   r   r   r   boolr   r   ry   r   r   r   r   r   r   r   r	   r   r|   r   r  dictr  r#   r#   r#   r$   r     sj    	
 	
	 $	
&
r   r/   )/r   dataclassesr   r   r   r3   collectionsr   
contextlibr   pathlibr   typingr   r   r   r   r	   r
   r   r   numpyr   r   packaging.versionr   r   	typeguardr   r   r   r   complexr   r   rK   rf   r%   r?   r   rF   r   	dataclassr!   r    rM   r   r#   r#   r#   r$   <module>   s8    $,


 #