o
    rri                     @   s   d Z ddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
 ddlmZ ddlmZ eeZdadadd	 ZdddZdd Zdd ZdddZde	dddZdS )z Distributed training utilities.
    N)DistributedSampler)
DataLoaderSubset)DistributedDataParallel)distrib   c                   C   s&   t j s	t  t at ad S )N)torchdistributedis_initializeddora_distribinitrank
world_size r   r   B/home/ubuntu/.local/lib/python3.10/site-packages/demucs/distrib.pyr      s   
r         ?c                 C   s   t | trtt|   \}}t||}tt||S tdkr!| S tjt	| dg dtj
d}||9 }tjj|tjjjd |d d |d     S )Nr   cudadevicedtype)op)
isinstancedictzipsorteditemsaverager   r   tensorlistfloat32r	   
all_reduceReduceOpSUMcpunumpytolist)metricscountkeysvaluesr   r   r   r   r       s   

 r   c                 C   s(   t dkr| S t| tj gtj dS )Nr   )
device_idsoutput_device)r   r   r   r   current_device)modelr   r   r   wrap-   s   
r/   c                   C   s   t dkrtj  d S d S )Nr   )r   r   r	   barrierr   r   r   r   r0   8   s   r0   c                 C   s   t dkr| S tjddtjd}t|krt| }t||d< tjj	||d t|kr:t
tj|tjd  }ntj|d  dtjd}tjj	||d t|kr^t|   } tdt|  | S )Nr   r   r   r   )src)r   zShared object of size )r   r   emptylongr   pickledumpslenr	   	broadcast
from_numpynp
frombufferuint8copyr   itemloadsr$   r%   tobytesloggerdebug)objr1   sizedumpbufferr   r   r   share=   s   
 rF   F)shuffleklassc                O   s   t dkr|| g|R d|i|S |r&t| }|| g|R i |d|iS t| tttt| t } || g|R d|i|S )z
    Create a dataloader properly in case of distributed training.
    If a gradient is going to be computed you must set `shuffle=True`.
    r   rG   sampler)r   r   r   r   ranger   r6   )datasetrG   rH   argskwargsrI   r   r   r   loaderT   s   rN   )r   )Nr   )__doc__loggingr4   r%   r9   r   torch.utils.data.distributedr   torch.utils.datar   r   torch.nn.parallel.distributedr   dorar   r   	getLogger__name__r@   r   r   r   r   r/   r0   rF   rN   r   r   r   r   <module>   s$   


