o
    پi                     @   s   d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	 e
eZdd Zdd
dZdd Zdd ZdddZdd Zdd Zdd Z			ddedee dee fddZdS ) zZ Distributed training/validation utils

Hacked together by / Copyright 2020 Ross Wightman
    N)Optional)distributed   )unwrap_modelc                 C   s&   |   }tj|tjjd || }|S )Nop)clonedist
all_reduceReduceOpSUM)tensornrt r   J/home/ubuntu/.local/lib/python3.10/site-packages/timm/utils/distributed.pyreduce_tensor   s   r   Fc                 C   sb   t | jddD ]&\}}d|v sd|v r.|r'tjj|tjjd |t| }qtj	|d qd S )NT)recurserunning_meanrunning_varr   r   )
r   named_bufferstorchr   r
   r	   r   r   float	broadcast)model
world_sizereducebn_namebn_bufr   r   r   distribute_bn   s   r   c                 C   
   | j dkS Nr   )rankargsr   r   r   is_global_primary%      
r%   c                 C   r    r!   )
local_rankr#   r   r   r   is_local_primary)   r&   r(   c                 C   s   |rt | S t| S )N)r(   r%   )r$   localr   r   r   
is_primary-   s   r*   c                   C   s<   dt jv rtt jd dkS dt jv rtt jd dkS dS )N
WORLD_SIZEr   SLURM_NTASKSFosenvironintr   r   r   r   is_distributed_env1   s
   

r1   c                  C   s   d} dD ]}|t jv rtt j| }  nqd}dD ]}|t jv r)tt j| } nqd}dD ]}|t jv r>tt j| } nq.| ||fS )Nr   )
LOCAL_RANKMPI_LOCALRANKIDSLURM_LOCALIDOMPI_COMM_WORLD_LOCAL_RANK)RANKPMI_RANKSLURM_PROCIDOMPI_COMM_WORLD_RANKr   )r+   PMI_SIZEr,   OMPI_COMM_WORLD_SIZEr-   )r'   vglobal_rankr   r   r   r   world_info_from_env9   s&   



r>   c                 C   s   d| _ d| _d| _d| _tt| ddt| dd t| dd d}|d | _|d	 | _|d
 | _|d | _|d | _ t| j}|S )NFr   r   devicecudadist_backenddist_url)r?   rA   rB   r   r=   r'   r   )r   r   r"   r'   init_distributed_device_sogetattrr?   r   )r$   resultr?   r   r   r   init_distributed_deviceO   s    







rF   r@   r?   rA   rB   c                 C   s  d}d}d}d}| j ddd^}}|d u r"ddddd	}	|	|d
}|p%d}t rndtjv rTt \}}}t|tjd< t|tjd< t|tjd< tjj	||||d nt \}}
}
tjj	||d tj
 }tj }d}|dkrtj sJ d|  d|dkrtj sJ d|  d|r| dkr|rtd|d  d|  d | d| } | drtj|  t| ||||dS )NFr   r   :)maxsplitcclhcclnccl)xpuhpur@   npugloozenv://r8   r2   r6   r+   )backendinit_methodr   r"   )rP   rQ   Tr@   zCUDA is not available but z was specified.rN   z Ascend NPU is not available but cpuzdevice index z removed from specified (z).zcuda:)r?   r=   r'   r   r   )splitgetr1   r.   r/   r>   strr   r   init_process_groupget_world_sizeget_rankr@   is_availablerN   _loggerwarning
startswith
set_devicedict)r?   rA   rB   r   r   r=   r'   device_type
device_idxdist_backends_r   r   r   rC   d   sd   



rC   )F)r@   NN)__doc__loggingr.   typingr   r   r   r	   r   r   	getLogger__name__rZ   r   r   r%   r(   r*   r1   r>   rF   rU   rC   r   r   r   r   <module>   s4    


