o
    Ti                     @   s(   d dl Z d dl mZ G dd deZdS )    N)ABCc                   @   s  e Zd Zdd Zejdd Zejdd Zejdd Zejd	d
 Z	ejdd Z
ejdd Zejdd Zejdd Zejdd Zejdd ZejdddZejdd ZejdddZejdddZejd d! Zejd"d# Zejd$d% Zejd&d' Zeejd(d) Zejd*d+ Zejdd,d-Zejdd.d/Zeejd0d1 Zejd2d3 Zejdd4d5Zejdd6d7Z ejdd8d9Z!ejdd:d;Z"ejdd<d=Z#ejdd>d?Z$ejdd@dAZ%ejddBdCZ&ejddDdEZ'ejddFdGZ(ejddHdIZ)ejddJdKZ*ejdLdM Z+ejdNdO Z,ejdPdQ Z-ejdRdS Z.ejdTdU Z/ejdVdW Z0ejdXdY Z1ejdZd[ Z2ejd\d] Z3ejd^d_ Z4ejd`da Z5ejddbdcZ6ejddde Z7eejdfdg Z8eejdhdi Z9eejdjdk Z:eejdldm Z;eejdndo Z<eejdpdq Z=eejdrds Z>ejddudvZ?ejdwdx Z@ejdydz ZAejd{d| ZBejd}d~ ZCejdd ZDejdd ZEejdd ZFejdd ZGejdd ZHejdd ZIejdd ZJdS )DeepSpeedAcceleratorc                 C   s   d | _ d | _d | _d S N)_name_communication_backend_name_compile_backendself r
   ^/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/accelerator/abstract_accelerator.py__init__   s   
zDeepSpeedAccelerator.__init__c                 C      d S r   r
   r   r
   r
   r   is_synchronized_device      z+DeepSpeedAccelerator.is_synchronized_devicec                 C   r   r   r
   r   r
   r
   r   use_host_timers   r   z$DeepSpeedAccelerator.use_host_timersc                 C   r   r   r
   r   r
   r
   r   resolves_data_dependency   r   z-DeepSpeedAccelerator.resolves_data_dependencyc                 C   r   r   r
   r   r
   r
   r   handles_memory_backpressure   r   z0DeepSpeedAccelerator.handles_memory_backpressurec                 C   r   r   r
   r	   device_indexr
   r
   r   device_name"   r   z DeepSpeedAccelerator.device_namec                 C   r   r   r
   r   r
   r
   r   device&   r   zDeepSpeedAccelerator.devicec                 C   r   r   r
   r   r
   r
   r   
set_device*   r   zDeepSpeedAccelerator.set_devicec                 C   r   r   r
   r   r
   r
   r   current_device.   r   z#DeepSpeedAccelerator.current_devicec                 C   r   r   r
   r   r
   r
   r   current_device_name2   r   z(DeepSpeedAccelerator.current_device_namec                 C   r   r   r
   r   r
   r
   r   device_count6   r   z!DeepSpeedAccelerator.device_countNc                 C   r   r   r
   r   r
   r
   r   synchronize:   r   z DeepSpeedAccelerator.synchronizec                 C   r   r   r
   r   r
   r
   r   random?   r   zDeepSpeedAccelerator.randomc                 C   r   r   r
   )r	   	new_stater   r
   r
   r   set_rng_stateC   r   z"DeepSpeedAccelerator.set_rng_statec                 C   r   r   r
   r   r
   r
   r   get_rng_stateG   r   z"DeepSpeedAccelerator.get_rng_statec                 C   r   r   r
   r	   seedr
   r
   r   manual_seedK   r   z DeepSpeedAccelerator.manual_seedc                 C   r   r   r
   r    r
   r
   r   manual_seed_allO   r   z$DeepSpeedAccelerator.manual_seed_allc                 C   r   r   r
   r   r
   r
   r   initial_seedS   r   z!DeepSpeedAccelerator.initial_seedc                 C   r   r   r
   r   r
   r
   r   default_generatorW   r   z&DeepSpeedAccelerator.default_generatorc                 C   r   r   r
   r   r
   r
   r   Stream\      zDeepSpeedAccelerator.Streamc                 C   r   r   r
   )r	   streamr
   r
   r   r(   a   r   zDeepSpeedAccelerator.streamc                 C   r   r   r
   r   r
   r
   r   current_streame   r   z#DeepSpeedAccelerator.current_streamc                 C   r   r   r
   r   r
   r
   r   default_streami   r   z#DeepSpeedAccelerator.default_streamc                 C   r   r   r
   r   r
   r
   r   Eventm   r'   zDeepSpeedAccelerator.Eventc                 C   r   r   r
   r   r
   r
   r   empty_caches   r   z DeepSpeedAccelerator.empty_cachec                 C   r   r   r
   r   r
   r
   r   memory_allocatedw   r   z%DeepSpeedAccelerator.memory_allocatedc                 C   r   r   r
   r   r
   r
   r   max_memory_allocated{   r   z)DeepSpeedAccelerator.max_memory_allocatedc                 C   r   r   r
   r   r
   r
   r   reset_max_memory_allocated   r   z/DeepSpeedAccelerator.reset_max_memory_allocatedc                 C   r   r   r
   r   r
   r
   r   memory_cached   r   z"DeepSpeedAccelerator.memory_cachedc                 C   r   r   r
   r   r
   r
   r   max_memory_cached   r   z&DeepSpeedAccelerator.max_memory_cachedc                 C   r   r   r
   r   r
   r
   r   reset_max_memory_cached   r   z,DeepSpeedAccelerator.reset_max_memory_cachedc                 C   r   r   r
   r   r
   r
   r   memory_stats   r   z!DeepSpeedAccelerator.memory_statsc                 C   r   r   r
   r   r
   r
   r   reset_peak_memory_stats   r   z,DeepSpeedAccelerator.reset_peak_memory_statsc                 C   r   r   r
   r   r
   r
   r   memory_reserved   r   z$DeepSpeedAccelerator.memory_reservedc                 C   r   r   r
   r   r
   r
   r   max_memory_reserved   r   z(DeepSpeedAccelerator.max_memory_reservedc                 C   r   r   r
   r   r
   r
   r   total_memory   r   z!DeepSpeedAccelerator.total_memoryc                 C   r   r   r
   r   r
   r
   r   available_memory   r   z%DeepSpeedAccelerator.available_memoryc                 C   r   r   r
   r   r
   r
   r   is_bf16_supported   r   z&DeepSpeedAccelerator.is_bf16_supportedc                 C   r   r   r
   r   r
   r
   r   is_fp16_supported   r   z&DeepSpeedAccelerator.is_fp16_supportedc                 C   r   r   r
   r   r
   r
   r   supported_dtypes   r   z%DeepSpeedAccelerator.supported_dtypesc                 C   r   r   r
   r   r
   r
   r   amp   r   zDeepSpeedAccelerator.ampc                 C   r   r   r
   r   r
   r
   r   is_available   r   z!DeepSpeedAccelerator.is_availablec                 C   r   r   r
   )r	   msgr
   r
   r   
range_push   r   zDeepSpeedAccelerator.range_pushc                 C   r   r   r
   r   r
   r
   r   	range_pop   r   zDeepSpeedAccelerator.range_popc                 C   r   r   r
   )r	   callbackr
   r
   r   	lazy_call   r   zDeepSpeedAccelerator.lazy_callc                 C   r   r   r
   r   r
   r
   r   communication_backend_name   r   z/DeepSpeedAccelerator.communication_backend_namec                 C   r   r   r
   r   r
   r
   r   is_triton_supported   r   z(DeepSpeedAccelerator.is_triton_supportedc                 C   r   r   r
   r   r
   r
   r   create_graph   r   z!DeepSpeedAccelerator.create_graphc                 C   r   r   r
   )r	   graphpoolr(   r
   r
   r   capture_to_graph   r   z%DeepSpeedAccelerator.capture_to_graphc                 C   r   r   r
   )r	   rF   r
   r
   r   replay_graph   r   z!DeepSpeedAccelerator.replay_graphc                 C   r   r   r
   r   r
   r
   r   BFloat16Tensor   r'   z#DeepSpeedAccelerator.BFloat16Tensorc                 C   r   r   r
   r   r
   r
   r   
ByteTensor   r'   zDeepSpeedAccelerator.ByteTensorc                 C   r   r   r
   r   r
   r
   r   DoubleTensor   r'   z!DeepSpeedAccelerator.DoubleTensorc                 C   r   r   r
   r   r
   r
   r   FloatTensor   r'   z DeepSpeedAccelerator.FloatTensorc                 C   r   r   r
   r   r
   r
   r   
HalfTensor   r'   zDeepSpeedAccelerator.HalfTensorc                 C   r   r   r
   r   r
   r
   r   	IntTensor   r'   zDeepSpeedAccelerator.IntTensorc                 C   r   r   r
   r   r
   r
   r   
LongTensor   r'   zDeepSpeedAccelerator.LongTensor   c                 C   r   r   r
   )r	   tensoralign_bytesr
   r
   r   
pin_memory  r   zDeepSpeedAccelerator.pin_memoryc                 C   r   r   r
   r	   rR   r
   r
   r   	is_pinned  r   zDeepSpeedAccelerator.is_pinnedc                 C   r   r   r
   rU   r
   r
   r   on_accelerator
  r   z#DeepSpeedAccelerator.on_acceleratorc                 C   r   r   r
   r   r
   r
   r   op_builder_dir  r   z#DeepSpeedAccelerator.op_builder_dirc                 C   r   r   r
   r	   
class_namer
   r
   r   create_op_builder  r   z&DeepSpeedAccelerator.create_op_builderc                 C   r   r   r
   rY   r
   r
   r   get_op_builder  r   z#DeepSpeedAccelerator.get_op_builderc                 C   r   r   r
   r   r
   r
   r   build_extension  r   z$DeepSpeedAccelerator.build_extensionc                 C   r   r   r
   r   r
   r
   r   export_envs   r   z DeepSpeedAccelerator.export_envsc                 C   r   r   r
   r   r
   r
   r   visible_devices_envs$  r   z)DeepSpeedAccelerator.visible_devices_envsc                 C   r   r   r
   )r	   current_envlocal_accelerator_idsr
   r
   r   set_visible_devices_envs(  r   z-DeepSpeedAccelerator.set_visible_devices_envsc                 C   r   r   r
   r   r
   r
   r   get_compile_backend,  r   z(DeepSpeedAccelerator.get_compile_backendc                 C   r   r   r
   )r	   backendr
   r
   r   set_compile_backend0  r   z(DeepSpeedAccelerator.set_compile_backendr   )NN)rQ   )K__name__
__module____qualname__r   abcabstractmethodr   r   r   r   r   r   r   r   r   r   r   r   r   r   r"   r#   r$   r%   propertyr&   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r?   r@   rB   rC   rD   rE   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rT   rV   rW   rX   r[   r\   r]   r^   r_   rb   rc   re   r
   r
   r
   r   r   
   s&   






































r   )ri   r   r   r
   r
   r
   r   <module>   s   