o
    Xεim
                     @   sx   d Z ddlmZ ddlZddlZddlmZ dd Zdd Zd	d
 Z	dd Z
dd Zdd Zdd Zdd Zdd ZdS )z
credit: https://github.com/BlackHC/toma/blob/master/toma/torch_cuda_memory.py

Helper to free Torch cuda memory and determine when a Torch exception might be
because of OOM conditions.
    )print_functionN)is_out_of_cpu_memoryc                   C   s$   t   tj rtj  dS dS )z$Gargage collect Torch (CUDA) memory.N)gccollecttorchcudais_availableempty_cache r
   r
   M/home/ubuntu/.local/lib/python3.10/site-packages/trainer/utils/cuda_memory.pygc_cuda   s   
r   c                   C   s   t j rt jdjS dS Nr   )r   r   r   get_device_propertiestotal_memoryr
   r
   r
   r   get_cuda_total_memory   s   
r   c                   C   s   t j rt t j  S dS r   )r   r   r   r   memory_reservedr
   r
   r
   r   !get_cuda_assumed_available_memory   s   
r   c                   C   s   t j rt t  S dS r   )r   r   r   r   get_cuda_blocked_memoryr
   r
   r
   r   get_cuda_available_memory#   s   
r   c               
   C   s   t j sdS t } | d }	 zt j|ft jdd}W n) tyC } zt|r8|d8 }|dkr7| W  Y d }~S n W Y d }~nd }~ww qd }t  | | S )Nr   i   Tr   )dtypedevice   @)	r   r   r   r   emptyuint8RuntimeErroris_cuda_out_of_memoryr   )available_memorycurrent_block_	exceptionr
   r
   r   r   *   s,   
r   c                 C   s.   t | ttjjfot| jdkod| jd v S )N   zCUDA out of memory.r   )
isinstancer   r   r   OutOfMemoryErrorlenargsr   r
   r
   r   r   @   s
   r   c                 C   s&   t | tot| jdkod| jd v S )Nr    z(cuDNN error: CUDNN_STATUS_NOT_SUPPORTED.r   )r!   r   r#   r$   r%   r
   r
   r   is_cudnn_snafuH   s
   
r&   c                   C   sZ   t j sd S tdt j d dt j d d tdt j d dt j d d d S )NzTotal:r   z GB Cached: GBz
Max Total:z GB Max Cached: )r   r   r   printmemory_allocatedr   max_memory_allocatedmax_memory_reservedr
   r
   r
   r   cuda_meminfoQ   s   
r,   c                 C   s   t | pt| pt| S )N)r   r&   r   r%   r
   r
   r   should_reduce_batch_sizea   s   r-   )__doc__
__future__r   r   r   trainer.utils.cpu_memoryr   r   r   r   r   r   r   r&   r,   r-   r
   r
   r
   r   <module>   s    	