o
    Xεi                     @   s   d dl Z d dlmZ d dlmZmZmZ d dlZd dlm	Z
 dd Zdd Zdd	 Zd
edefddZededdfddZededdddfddZdd Zdd ZdS )    N)wraps)AnyCallableOptionalc                   C   s   t  sdS t  sdS dS )NFT)distis_availableis_initialized r	   r	   M/home/ubuntu/.local/lib/python3.10/site-packages/trainer/utils/distributed.pyis_dist_avail_and_initialized
   s
   r   c                  C   s2   d} | D ]}t j|}|d urt|  S qdS )N)RANK
LOCAL_RANKSLURM_PROCIDJSM_NAMESPACE_RANKr   )osenvirongetint)	rank_keyskeyrankr	   r	   r
   get_rank   s   r   c                   C   s
   t  dkS )Nr   )r   r	   r	   r	   r
   is_main_process   s   
r   fnreturnc                    s*   t  dtdtdtt f fdd}|S )Nargskwargsr   c                     s   t  r
 | i |S d S N)r   )r   r   r   r	   r
   
wrapped_fn    s   z"rank_zero_only.<locals>.wrapped_fn)r   r   r   )r   r   r	   r   r
   rank_zero_only   s    r    messagec                 O   s   t |  d S r   )print)r!   r   r   r	   r	   r
   rank_zero_print)   s   r#   loggerLoggerc                 O   s   | |  d S r   )info)r!   r$   r   r   r	   r	   r
   rank_zero_logger_info.   s   r'   c                 C   s&   |   }tj|tjjd || }|S )N)op)cloner   
all_reduce	reduce_opSUM)tensornum_gpusrtr	   r	   r
   reduce_tensor3   s   r0   c                 C   s@   t j s	J dt j| t j   tj|||| |d d S )NzDistributed mode requires CUDA.)init_method
world_sizer   
group_name)torchcudar   
set_devicedevice_countr   init_process_group)r   r.   r3   dist_backenddist_urlr	   r	   r
   init_distributed:   s   
r;   )r   	functoolsr   typingr   r   r   r4   torch.distributeddistributedr   r   r   r   r    strr#   r'   r0   r;   r	   r	   r	   r
   <module>   s   	
