o
    oi(                     @   s   d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	 d dl
Z
d dlmZ d dlZd dlmZ d dlmZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ e eZG d
d deZ dedee!e"f fddZ#de$de!fddZ%dS )    N)AnyDictListOptionalUnion)override)_AcceleratorRegistry)_check_cuda_matmul_precision_clear_cuda_memorynum_cuda_devices_parse_gpu_ids)_DEVICE)Accelerator)MisconfigurationExceptionc                   @   s  e Zd ZdZedejddfddZedd	d
Ze	de
ddfddZededeeef fddZedddZe	edee
eee
 f deee
  fddZe	edee
 deej fddZe	ede
fddZe	edefddZeededdfddZdS ) CUDAAcceleratorz$Accelerator for NVIDIA CUDA devices.devicereturnNc                 C   s2   |j dkrtd| dt| tj| dS )zs
        Raises:
            MisconfigurationException:
                If the selected device is not GPU.
        cudazDevice should be GPU, got z insteadN)typer   r	   torchr   
set_deviceselfr    r   a/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/pytorch_lightning/accelerators/cuda.pysetup_device%   s   
zCUDAAccelerator.setup_devicetrainer
pl.Trainerc                 C   s   |  |j t  d S N)set_nvidia_flags
local_rankr
   )r   r   r   r   r   setup1   s   
zCUDAAccelerator.setupr!   c                 C   sL   dt jd< ddd tt D }t d|}td|  d| d	 d S )
N
PCI_BUS_IDCUDA_DEVICE_ORDER,c                 s       | ]}t |V  qd S r   str.0xr   r   r   	<genexpr>;       z3CUDAAccelerator.set_nvidia_flags.<locals>.<genexpr>CUDA_VISIBLE_DEVICESzLOCAL_RANK: z - CUDA_VISIBLE_DEVICES: [])osenvironjoinranger   getenv_loginfo)r!   all_gpu_idsdevicesr   r   r   r    7   s   
z CUDAAccelerator.set_nvidia_flagsc                 C   s   t j|S )a(  Gets stats for the given GPU device.

        Args:
            device: GPU device for which to get stats

        Returns:
            A dictionary mapping the metrics to their values.

        Raises:
            FileNotFoundError:
                If nvidia-smi installation not found

        )r   r   memory_statsr   r   r   r   get_device_stats?   s   z CUDAAccelerator.get_device_statsc                 C   s
   t   d S r   )r
   )r   r   r   r   teardownP   s   
zCUDAAccelerator.teardownr8   c                 C   s   t | ddS )z!Accelerator device parsing logic.T)include_cudar   r8   r   r   r   parse_devicesT   s   zCUDAAccelerator.parse_devicesc                 C   s   dd | D S )z*Gets parallel devices for the Accelerator.c                 S   s   g | ]}t d |qS )r   )r   r   r*   ir   r   r   
<listcomp>^   s    z8CUDAAccelerator.get_parallel_devices.<locals>.<listcomp>r   r=   r   r   r   get_parallel_devicesZ   s   z$CUDAAccelerator.get_parallel_devicesc                   C   s   t  S )z!Get the devices when set to auto.r   r   r   r   r   auto_device_count`   s   z!CUDAAccelerator.auto_device_countc                   C   s
   t  dkS )Nr   rC   r   r   r   r   is_availablef   s   
zCUDAAccelerator.is_availableaccelerator_registryc                 C   s   |j d| | jd d S )Nr   )description)register__name__)clsrF   r   r   r   register_acceleratorsk   s
   
z%CUDAAccelerator.register_accelerators)r   r   r   N)r   N)rI   
__module____qualname____doc__r   r   r   r   r"   staticmethodintr    r   r   r(   r   r:   r;   r   r   r   r>   rB   rD   boolrE   classmethodr   rK   r   r   r   r   r   "   s6    , r   r   r   c           
         s   t d}|du rtdg d}dd |D }d|}tj| }t|}tj	|d| d	d
| gdddd}dt
dtfdd |j } fdd|dD }	dd t||	D S )a/  Get GPU stats including memory, fan speed, and temperature from nvidia-smi.

    Args:
        device: GPU device for which to get stats

    Returns:
        A dictionary mapping the metrics to their values.

    Raises:
        FileNotFoundError:
            If nvidia-smi installation not found

    z
nvidia-smiNznvidia-smi: command not found))zutilization.gpu%)zmemory.usedMB)zmemory.freerT   )zutilization.memoryrS   )z	fan.speedrS   )ztemperature.gpu   °C)ztemperature.memoryrU   c                 S   s   g | ]\}}|qS r   r   )r*   k_r   r   r   rA          z(get_nvidia_gpu_stats.<locals>.<listcomp>r%   z--query-gpu=z--format=csv,nounits,noheaderz--id=zutf-8T)encodingcapture_outputcheckr+   r   c                 S   s    zt | W S  ty   Y dS w )Ng        )float
ValueError)r+   r   r   r   	_to_float   s
   
z'get_nvidia_gpu_stats.<locals>._to_floatc                    s   g | ]} |qS r   r   r)   r^   r   r   rA      rX   z, c                 S   s&   i | ]\\}}}| d | d|qS )z ()r   )r*   r+   unitstatr   r   r   
<dictcomp>   s   & z(get_nvidia_gpu_stats.<locals>.<dictcomp>)shutilwhichFileNotFoundErrorr2   r   _utils_get_device_index_get_gpu_id
subprocessrunr(   r\   stdoutstripsplitzip)
r   nvidia_smi_pathgpu_stat_metricsgpu_stat_keys	gpu_queryindexgpu_idresultsstatsr   r_   r   get_nvidia_gpu_statsu   s$   
	

ry   	device_idc                 C   s:   d dd tt D }tjd|dd}||   S )zGet the unmasked real GPU IDs.r%   c                 s   r&   r   r'   r?   r   r   r   r,      r-   z_get_gpu_id.<locals>.<genexpr>r.   )default)r2   r3   r   r0   r4   rn   rm   )rz   r{   cuda_visible_devicesr   r   r   ri      s   ri   )&loggingr0   rd   rj   typingr   r   r   r   r   r   typing_extensionsr   pytorch_lightningpllightning_fabric.acceleratorsr   "lightning_fabric.accelerators.cudar	   r
   r   (lightning_fabric.utilities.device_parserr    lightning_fabric.utilities.typesr   *pytorch_lightning.accelerators.acceleratorr   &pytorch_lightning.utilities.exceptionsr   	getLoggerrI   r5   r   r(   r\   ry   rP   ri   r   r   r   r   <module>   s$   
S2