o
    ni                     @   s   d dl Z d dlmZ d dlmZmZmZmZmZm	Z	m
Z
mZ zd dlZW n ey/   dZY nw d dlZddlmZ ddlmZ ddlmZmZmZ erXd dlmZ d d	lmZ G d
d dZejG dd dZdS )    N)deque)TYPE_CHECKINGAnyDictListOptionalSetTupleUnion   )aggregate_mean)asset_registry)	InterfaceMetricMetricsMonitor)Deque)SettingsStaticc                   @   s   e Zd ZU dZdZded< h dZddedee	 d	dfd
dZ
ededed	eeeeeef f  fddZdddZdddZd	efddZdS )IPUStatsz Stats for Graphcore IPU devices.z	ipu.{}.{}zDeque[dict]samples>   	ipu poweripu utilisationaverage die tempaverage board tempipu utilisation (session)clockNpidgc_ipu_inforeturnc                 C   sL   t  | _|d u rtstdt | _n|| _| jd || _t | _d S )Nz7Monitoring IPU stats requires gcipuinfo to be installedT)	r   r   	gcipuinfoImportError_gc_ipu_infosetUpdateMode_pidset_devices_called)selfr   r    r&   b/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/wandb/sdk/internal/system/assets/ipu.py__init__(   s   zIPUStats.__init__keyvaluec                 C   s   ddddddd}|  D ]\}}| |r,||r,|d t|  }|  d| d} qzt|}| r:t|n|}W | |fS  tyJ   Y d S w )	NCMHzW%zGT/s)tempr   powerutilisationzutilisation (session)speedz ())itemsendswithlenfloat
is_integerint
ValueError)r)   r*   metric_suffixesmetricsuffixfloat_value	num_valuer&   r&   r'   parse_metric9   s&   	zIPUStats.parse_metricc              
   C   s  zii }| j  }|D ]V}t|}|d}|d u s t|| jkr!q
|d}|| jv}|d ur5| j| | D ]&\}}	|pC|| j	v }
|
sGq9| 
||	}|d u rRq9|\}}||| j||< q9q
| j| W d S  ty } ztjd| dd W Y d }~d S d }~ww )Nzuser process ididzIPU stats error F)repeat)r    
getDevicesdictgetr9   r"   r$   addr4   variable_metric_keysr@   nameformatr   append	Exceptionwandbtermwarn)r%   statsdevicesdevicedevice_metricsr   	device_idinitial_callr)   r*   
log_metricparsed
parsed_keyparsed_valueer&   r&   r'   sampleQ   s6   




"zIPUStats.samplec                 C      | j   d S N)r   clearr%   r&   r&   r'   r\   q      zIPUStats.clearc                    sJ   | j si S i }| j d  D ]  fdd| j D }t|}|| < q|S )Nr   c                    s   g | ]
} |v r|  qS r&   r&   ).0sr)   r&   r'   
<listcomp>y   s    z&IPUStats.aggregate.<locals>.<listcomp>)r   keysr   )r%   rN   r   	aggregater&   ra   r'   rd   t   s   
zIPUStats.aggregater[   r   N)__name__
__module____qualname____doc__rH   __annotations__rG   r9   r   r   r(   staticmethodstrr	   r
   r7   r@   rY   r\   rD   rd   r&   r&   r&   r'   r      s   
 	,

 r   c                   @   s\   e Zd Zdddddejddfdd	Zedefd
dZdddZ	dddZ
defddZdS )IPU	interfacer   settingsr   shutdown_eventr   Nc                 C   s6   | j j | _t|jg| _t| j| j|||| _d S r[   )		__class__rf   lowerrH   r   x_stats_pidmetricsr   metrics_monitor)r%   rn   ro   rp   r&   r&   r'   r(      s   
zIPU.__init__c                 C   s   t d uS r[   )r   )clsr&   r&   r'   is_available   s   zIPU.is_availablec                 C   rZ   r[   )ru   startr]   r&   r&   r'   rx      r^   z	IPU.startc                 C   rZ   r[   )ru   finishr]   r&   r&   r'   ry      r^   z
IPU.finishc                 C   st   | j d j }t|}g }t|D ]\}}t|}||dp"||d|dp,dd q| j||ddiS )	Nr   rA   board ipu index
board typeunknown)rA   rz   r{   	Graphcore)device_countrO   vendor)	rt   r    rC   r6   	enumeraterD   rJ   rE   rH   )r%   device_datar~   rO   irP   rQ   r&   r&   r'   probe   s"   	z	IPU.probere   )rf   rg   rh   	threadingEventr(   classmethodboolrw   rx   ry   rD   r   r&   r&   r&   r'   rm      s    


rm   )r   collectionsr   typingr   r   r   r   r   r   r	   r
   r   r   rL   aggregatorsr   r   
interfacesr   r   r   r   "wandb.sdk.internal.settings_staticr   r   registerrm   r&   r&   r&   r'   <module>   s$    (i