o
    ni5                     @   sB  d dl Z d dlZd dlmZ d dlmZmZ zd dlZW n ey'   dZY nw d dl	m
Z
 ddlmZ ddlmZ ddlmZmZmZ erTd d	lmZ d d
lmZ eZe eZdddedefddZG dd dZG dd dZG dd dZG dd dZ G dd dZ!G dd dZ"G dd dZ#ej$G dd  d Z%dS )!    N)deque)TYPE_CHECKINGList)pynvml   )aggregate_mean)asset_registry)	InterfaceMetricMetricsMonitor)Deque)SettingsStatic
gpu_handle	GPUHandlepidreturnc                 C   s   t d u rdS zt j|d}W n t jy   Y dS w |jdd}|| dd |D }dd t| D }dd t| D }||B }t||@ d	kS )
NFr   T)	recursivec                 S      h | ]}|j qS  r   .0processr   r   b/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/wandb/sdk/internal/system/assets/gpu.py	<setcomp>)   s    z-gpu_in_use_by_this_process.<locals>.<setcomp>c                 S   r   r   r   r   r   r   r   r   +       c                 S   r   r   r   r   r   r   r   r   /   r   r   )	psutilProcessNoSuchProcesschildrenappendr   $nvmlDeviceGetComputeRunningProcesses%nvmlDeviceGetGraphicsRunningProcesseslen)r   r   base_processour_processesour_pidscompute_pidsgraphics_pidspids_using_devicer   r   r   gpu_in_use_by_this_process   s$   
r*   c                   @   R   e Zd ZU dZdZded< deddfdd	Zdd
dZdddZ	de
fddZdS )GPUMemoryUtilizationz/GPU memory utilization in percent for each GPU.zgpu.{}.memoryDeque[List[float]]samplesr   r   Nc                 C      || _ tg | _d S Nr   r   r.   selfr   r   r   r   __init__A      zGPUMemoryUtilization.__init__c                 C   F   g }t  }t|D ]}t |}|t |j q
| j| d S r0   )r   nvmlDeviceGetCountrangenvmlDeviceGetHandleByIndexr    nvmlDeviceGetUtilizationRatesmemoryr.   )r3   memory_utilization_ratedevice_countihandler   r   r   sampleE      

zGPUMemoryUtilization.samplec                 C      | j   d S r0   r.   clearr3   r   r   r   rD   O      zGPUMemoryUtilization.clearc                       | j si S i }t }t|D ].  fdd| j D }t|}||| j < t }t|| j	r=||| jd  < q|S )Nc                       g | ]}|  qS r   r   r   r@   r>   r   r   
<listcomp>X       z2GPUMemoryUtilization.aggregate.<locals>.<listcomp>process.
r.   r   r7   r8   r   nameformatr9   r*   r   r3   statsr=   r.   	aggregater?   r   rJ   r   rS   R      
zGPUMemoryUtilization.aggregater   N__name__
__module____qualname____doc__rO   __annotations__intr4   r@   rD   dictrS   r   r   r   r   r,   9      
 


r,   c                   @   r+   )GPUMemoryAllocatedz-GPU memory allocated in percent for each GPU.zgpu.{}.memoryAllocatedr-   r.   r   r   Nc                 C   r/   r0   r1   r2   r   r   r   r4   k   r5   zGPUMemoryAllocated.__init__c                 C   sT   g }t  }t|D ]}t |}t |}||j|j d  q
| j| d S Nd   )	r   r7   r8   r9   nvmlDeviceGetMemoryInfor    usedtotalr.   r3   memory_allocatedr=   r>   r?   memory_infor   r   r   r@   o   s   

zGPUMemoryAllocated.samplec                 C   rB   r0   rC   rE   r   r   r   rD   x   rF   zGPUMemoryAllocated.clearc                    rG   )Nc                    rH   r   r   rI   rJ   r   r   rK      rL   z0GPUMemoryAllocated.aggregate.<locals>.<listcomp>rM   rN   rQ   r   rJ   r   rS   {   rT   zGPUMemoryAllocated.aggregaterU   rV   r   r   r   r   r_   c      
 

	r_   c                   @   r+   )GPUMemoryAllocatedBytesz+GPU memory allocated in bytes for each GPU.zgpu.{}.memoryAllocatedBytesr-   r.   r   r   Nc                 C   r/   r0   r1   r2   r   r   r   r4      r5   z GPUMemoryAllocatedBytes.__init__c                 C   sJ   g }t  }t|D ]}t |}t |}||j q
| j| d S r0   )r   r7   r8   r9   rb   r    rc   r.   re   r   r   r   r@      s   

zGPUMemoryAllocatedBytes.samplec                 C   rB   r0   rC   rE   r   r   r   rD      rF   zGPUMemoryAllocatedBytes.clearc                    rG   )Nc                    rH   r   r   rI   rJ   r   r   rK      rL   z5GPUMemoryAllocatedBytes.aggregate.<locals>.<listcomp>rM   rN   rQ   r   rJ   r   rS      rT   z!GPUMemoryAllocatedBytes.aggregaterU   rV   r   r   r   r   ri      rh   ri   c                   @   r+   )GPUUtilizationz(GPU utilization in percent for each GPU.z
gpu.{}.gpur-   r.   r   r   Nc                 C   r/   r0   r1   r2   r   r   r   r4      r5   zGPUUtilization.__init__c                 C   r6   r0   )r   r7   r8   r9   r    r:   gpur.   )r3   gpu_utilization_rater=   r>   r?   r   r   r   r@      rA   zGPUUtilization.samplec                 C   rB   r0   rC   rE   r   r   r   rD      rF   zGPUUtilization.clearc                    rG   )Nc                    rH   r   r   rI   rJ   r   r   rK      rL   z,GPUUtilization.aggregate.<locals>.<listcomp>rM   rN   rQ   r   rJ   r   rS      rT   zGPUUtilization.aggregaterU   rV   r   r   r   r   rj      r^   rj   c                   @   r+   )GPUTemperaturez(GPU temperature in Celsius for each GPU.zgpu.{}.tempr-   r.   r   r   Nc                 C   r/   r0   r1   r2   r   r   r   r4      r5   zGPUTemperature.__init__c                 C   sH   g }t  }t|D ]}t |}|t |t j q
| j| d S r0   )r   r7   r8   r9   r    nvmlDeviceGetTemperatureNVML_TEMPERATURE_GPUr.   )r3   temperaturer=   r>   r?   r   r   r   r@      s   
zGPUTemperature.samplec                 C   rB   r0   rC   rE   r   r   r   rD      rF   zGPUTemperature.clearc                    rG   )Nc                    rH   r   r   rI   rJ   r   r   rK     rL   z,GPUTemperature.aggregate.<locals>.<listcomp>rM   rN   rQ   r   rJ   r   rS      rT   zGPUTemperature.aggregaterU   rV   r   r   r   r   rm      s   
 

rm   c                   @   r+   )GPUPowerUsageWattsz&GPU power usage in Watts for each GPU.zgpu.{}.powerWattsr-   r.   r   r   Nc                 C   r/   r0   r1   r2   r   r   r   r4     r5   zGPUPowerUsageWatts.__init__c                 C   sL   g }t  }t|D ]}t |}t |d }|| q
| j| d S )Ni  )r   r7   r8   r9   nvmlDeviceGetPowerUsager    r.   )r3   power_usager=   r>   r?   power_wattsr   r   r   r@     s   
zGPUPowerUsageWatts.samplec                 C   rB   r0   rC   rE   r   r   r   rD      rF   zGPUPowerUsageWatts.clearc                    sv   i }t  }t|D ].  fdd| jD }t|}||| j < t  }t|| j	r8||| jd  < q
|S )Nc                    rH   r   r   rI   rJ   r   r   rK   '  rL   z0GPUPowerUsageWatts.aggregate.<locals>.<listcomp>rM   )
r   r7   r8   r.   r   rO   rP   r9   r*   r   rQ   r   rJ   r   rS   #  s   
zGPUPowerUsageWatts.aggregaterU   rV   r   r   r   r   rq     s   
 

	rq   c                   @   r+   )GPUPowerUsagePercentz(GPU power usage in percent for each GPU.zgpu.{}.powerPercentr-   r.   r   r   Nc                 C   r/   r0   r1   r2   r   r   r   r4   9  r5   zGPUPowerUsagePercent.__init__c                 C   sZ   g }t  }t|D ]}t |}t |}t |}||| d  q
| j| d S r`   )r   r7   r8   r9   rr   nvmlDeviceGetEnforcedPowerLimitr    r.   )r3   rs   r=   r>   r?   rt   power_capacity_wattsr   r   r   r@   =  s   


zGPUPowerUsagePercent.samplec                 C   rB   r0   rC   rE   r   r   r   rD   G  rF   zGPUPowerUsagePercent.clearc                    rG   )Nc                    rH   r   r   rI   rJ   r   r   rK   P  rL   z2GPUPowerUsagePercent.aggregate.<locals>.<listcomp>rM   rN   rQ   r   rJ   r   rS   J  rT   zGPUPowerUsagePercent.aggregaterU   rV   r   r   r   r   ru   2  s   
 


ru   c                   @   s\   e Zd Zdddddejddfdd	Zedefd
dZdddZ	dddZ
defddZdS )GPU	interfacer	   settingsr   shutdown_eventr   Nc                 C   sf   | j j | _t|jt|jt|jt|jt	|jt
|jt|jg| _t| j| j|||| _d S r0   )	__class__rW   lowerrO   r_   x_stats_pidri   r,   rj   rm   rq   ru   metricsr   metrics_monitor)r3   ry   rz   r{   r   r   r   r4   ]  s    	
zGPU.__init__c              
   C   sZ   zt   W dS  t jy   Y dS  ty, } ztd|  W Y d }~dS d }~ww )NTFzError initializing NVML: )r   nvmlInitNVMLError_LibraryNotFound	Exceptionloggererror)clser   r   r   is_availableu  s   zGPU.is_availablec                 C   rB   r0   )r   startrE   r   r   r   r     rF   z	GPU.startc                 C   rB   r0   )r   finishrE   r   r   r   r     rF   z
GPU.finishc              
   C   s   i }z>t   t t d|d< t  |d< t  }g }t|D ]}t |}t |}|t ||jd q!||d< W |S  t j	yK   Y |S  t
ye } ztd|  W Y d }~|S d }~ww )Nr   rk   	gpu_count)rO   memory_totalgpu_deviceszError Probing GPU: )r   r   nvmlDeviceGetNamer9   r7   r8   rb   r    rd   	NVMLErrorr   r   r   )r3   infor=   devicesr>   r?   gpu_infor   r   r   r   probe  s2   


z	GPU.proberU   )rW   rX   rY   	threadingEventr4   classmethodboolr   r   r   r]   r   r   r   r   r   rx   [  s    



rx   )&loggingr   collectionsr   typingr   r   r   ImportErrorwandb.vendor.pynvmlr   aggregatorsr   r   
interfacesr	   r
   r   r   "wandb.sdk.internal.settings_staticr   objectr   	getLoggerrW   r   r\   r   r*   r,   r_   ri   rj   rm   rq   ru   registerrx   r   r   r   r   <module>   s8    
*))*-&)