o
    ίi                     @   s&  d dl Z d dlZd dlmZ d dlmZmZ d dlmZmZ d dl	m
Z
 d dlmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlZd dlmZ eeef Ze ZeG dd	 d	ZeG d
d dZ G dd dZ!edg dZ"G dd dZ#	dddZ$dej%defddZ&dd Z'dS )    N)
namedtuple)asdict	dataclass)datetimetimezonePath)AnyDictListOptionalUnion)get_is_masterc                   @   s~  e Zd ZU dZee ed< dZee ed< dZee ed< dZ	ee ed< dZ
ee ed< dZee ed< dZee ed< dZee ed	< dZeee  ed
< dZeee  ed< dZee ed< dZee ed< dZee ed< dZeeeef  ed< dZee ed< dZee ed< dZee ed< dZee ed< dZee ed< dZee ed< dZee ed< dZee ed< dS )	WandbArgsNjob_typedirprojectentitytagsgroupnamenotesconfig_exclude_keysconfig_include_keys	anonymousmodeallow_val_changeresumeforcetensorboardsync_tensorboardmonitor_gym	save_codeid	fork_fromresume_from)__name__
__module____qualname__r   r   str__annotations__r   r   r   r   r   r   r   r   r   r   r   r   r   boolr   r   r   r   r    r!   r"   r#   r$   r%    r,   r,   @/home/ubuntu/.local/lib/python3.10/site-packages/core/metrics.pyr      s.   
 r   c                   @   sF   e Zd ZU dZeed< dZeed< dZe	e ed< dZ
e	e ed< dS )LoggingArgs
   freqINFOlevelNacc_freqwandb)r&   r'   r(   r0   intr*   r2   r)   r3   r   r4   r   r,   r,   r,   r-   r.   2   s
   
 r.   c                   @   sb   e Zd Zddedee fddZdd Zdee	ef fd	d
Z
dd Zdd Zdd Zdd ZdS )MetricLoggerNoutdirargsc                 C   s   || _ d | _|| _d S N)r7   jsonl_writerr8   )selfr7   r8   r,   r,   r-   __init__<   s   
zMetricLogger.__init__c                 C   sj   | j d u rt| jd| _ | jd ur/| jjjd ur1t r3tjddt| jit| jjj}d S d S d S d S )Naconfigr,   )	r:   openr7   r8   loggingr4   r   initr   )r;   runr,   r,   r-   r?   A   s   


zMetricLogger.openmetricsc                 C   sh   | j d ur| j jjd urtjd urtj||d d |dttj	
 i tt|| jdd d S )Nglobal_stepstep
created_atT)fileflush)r8   r@   r4   rB   logupdater   nowr   utc	isoformatprintjsondumpsr:   )r;   rC   r,   r,   r-   rJ   N   s   

zMetricLogger.logc                 C   s"   | j d ur| j   d | _ d S d S r9   )r:   closer;   r,   r,   r-   rR   Y   s   


zMetricLogger.closec                 C   s   |    | S r9   )r?   rS   r,   r,   r-   	__enter__^   s   zMetricLogger.__enter__c                 C      |    d S r9   rR   )r;   exc_type	exc_value	tracebackr,   r,   r-   __exit__b      zMetricLogger.__exit__c                 C   rU   r9   rV   rS   r,   r,   r-   __del__e   r[   zMetricLogger.__del__r9   )r&   r'   r(   r   r   r	   r<   r?   r
   r)   rJ   rR   rT   rZ   r\   r,   r,   r,   r-   r6   ;   s    r6   GPUMemStats)max_active_gibmax_active_pctmax_reserved_gibmax_reserved_pctnum_alloc_retriesnum_ooms
power_drawc                   @   sH   e Zd ZdZddefddZdd Zdd	 Zd
d Zdd Z	dd Z
dS )GPUMemoryMonitorz+
    Class to monitor GPU memory usage
    cuda:0devicec                 C   s`   t || _t j| j| _t j | _t j| jj| _	| 
| j	| _t j  t j  d S r9   )torchrg   cudaget_device_namedevice_namecurrent_devicedevice_indexget_device_propertiestotal_memorydevice_capacity_to_gibdevice_capacity_gibreset_peak_memory_statsempty_cache)r;   rg   r,   r,   r-   r<   |   s   
zGPUMemoryMonitor.__init__c                 C   s   d}|| }|S )Ni   @r,   )r;   memory_in_bytes_gib_in_bytesmemory_in_gibr,   r,   r-   rq      s   zGPUMemoryMonitor._to_gibc                 C   s   d| | j  S )Nd   )rp   )r;   memoryr,   r,   r-   _to_pct   s   zGPUMemoryMonitor._to_pctc                 C   s   t j| j}|d }| |}| |}|d }| |}| |}|d }|d }	t j }
|dkr<t| d |	dkrHt|	 d t	||||||	|
S )Nzactive_bytes.all.peakzreserved_bytes.all.peakrb   rc   r   z  CUDA memory allocation retries.z CUDA OOM errors thrown.)
rh   ri   memory_statsrg   rq   rz   rd   loggerwarningr]   )r;   	cuda_info
max_activer^   r_   max_reservedr`   ra   num_retriesrc   rd   r,   r,   r-   get_peak_stats   s.   




zGPUMemoryMonitor.get_peak_statsc                 C   s   t j  t j  d S r9   )rh   ri   rs   reset_accumulated_memory_statsrS   r,   r,   r-   reset_peak_stats   s   
z!GPUMemoryMonitor.reset_peak_statsc                 C   sB   |   }| j d| j d| j d}||j d|j d7 }| S )Nz (z): z GiB capacity, z GiB peak, z% peak)r   rk   rm   rr   r`   ra   )r;   	mem_statsdisplay_strr,   r,   r-   __str__   s   zGPUMemoryMonitor.__str__N)rf   )r&   r'   r(   __doc__r)   r<   rq   rz   r   r   r   r,   r,   r,   r-   re   w   s    re   
perceptioncodegen-teamTc                 C   sJ  dd l }ddlm} dd l}ddlm} ||| d }	||	}	|r^|j|	|	d ||d t	|| d }
|
D ]}|
|}|j||d d	 q:W d    n1 sUw   Y  |  |r|j|	|	d ||d t	|| d
 "}
|
D ]}|
|}|jdd | D |d d	 qvW d    n1 sw   Y  |  d S d S )Nr   r   )	OmegaConfzconfig.yamlr   )r>   r   r   r   zmetrics.jsonlrD   rE   zmetrics.eval.jsonlc                 S   s,   i | ]\}}d |v rd| d d |qS )/zevals/.)replace).0r   valuer,   r,   r-   
<dictcomp>   s
    z)upload_train_to_wandb.<locals>.<dictcomp>)rP   pathlibr   r4   	omegaconfr   loadto_containerrA   r?   loadsrJ   finishitems)ckpt_dirr   r   trainevalrP   r   r4   r   cfgflmr,   r,   r-   upload_train_to_wandb   s<   


r   modelreturnc                 C   s   dd |   D }t| S )zf
    Get the total model params
    Args : only_trainable: whether to only count trainable params
    c                 S   s   i | ]	\}}||  qS r,   )numel)r   npr,   r,   r-   r      s    z"get_num_params.<locals>.<dictcomp>)named_parameterssumvalues)r   r   r,   r,   r-   get_num_params   s   r   c              	      s   g g  d\}} fdd}t | tjjr+| D ]}||\}}||7 }||7 }qn|| \}}td td   td  td|| dd	|dd
|dd d S )Nr   r   c                    sV   d\}}|   D ]\}}|jr| || 7 }q | || 7 }q||fS )Nr   )r   requires_gradappendr   )_modelnum_frozen_paramsnum_unfrozen_paramspnamepvalfrozen_paramsunfrozen_paramsr,   r-   _fn   s   

zlog_model_params.<locals>._fnz.Logging Trainable Parameters after first step.zFrozen params: zTrainable params: zParams total: ,z, Learnable: z
, Frozen: r   )
isinstancerh   nn
ModuleListr|   infodebug)r   r   r   r   r   _num_frozen_params_num_unfrozen_paramsr,   r   r-   log_model_params   s(   


r   )r   r   TT)(rP   r@   collectionsr   dataclassesr   r   r   r   r   r   typingr	   r
   r   r   r   rh   torch.distributeddistributeddisttorch.nnr   r4   core.distributedr   r5   floatScalar	getLoggerr|   r   r.   r6   r]   re   r   Moduler   r   r,   r,   r,   r-   <module>   s8   .G
(	