o
    ϯiI                     @   s   d dl mZmZ d dlZd dlmZ d dlmZ G dd deZ	dej
dej
fdd	ZG d
d deZdej
deej
 fddZdS )    )ListTupleN)Functionc                   @   s@   e Zd ZedejdejfddZedejdejfddZdS )	
_AllReduceinputreturnc                    sF    fddt t D }tj| dd tj|dd}tj|ddS )Nc                       g | ]}t  qS  torch
zeros_like).0kr   r	   I/home/ubuntu/.local/lib/python3.10/site-packages/fvcore/nn/distributed.py
<listcomp>       z&_AllReduce.forward.<locals>.<listcomp>Fasync_opr   )dim)rangedistget_world_size
all_gatherr   stacksum)ctxr   
input_listinputsr	   r   r   forward   s   z_AllReduce.forwardgrad_outputc                 C   s   t j|dd |S )NFr   )r   
all_reduce)r   r    r	   r	   r   backward   s   z_AllReduce.backwardN)__name__
__module____qualname__staticmethodr   Tensorr   r"   r	   r	   r	   r   r      s
    r   r   r   c                 C   s*   t  rt  rt  dkr| S t| S )z:
    Differentiable counterpart of `dist.all_reduce`.
       )r   is_availableis_initializedr   r   applyr   r	   r	   r   differentiable_all_reduce   s   
r,   c                   @   sH   e Zd Zedejdeejdf fddZedejdejfddZd	S )

_AllGatherxr   .c                    s.    fddt t D }t|  t|S )Nc                    r   r	   r
   )r   _r.   r	   r   r   *   r   z&_AllGather.forward.<locals>.<listcomp>)r   r   r   r   tuple)r   r.   outputr	   r0   r   r   (   s   z_AllGather.forwardgradsc                 G   s    t |}t| |t  S )N)r   r   r   r!   get_rank)r   r3   all_gradientsr	   r	   r   r"   .   s   

z_AllGather.backwardN)	r#   r$   r%   r&   r   r'   r   r   r"   r	   r	   r	   r   r-   '   s
     r-   c                 C   s0   t  rt  rt  dkr| gS tt| S )z:
    Differentiable counterpart of `dist.all_gather`.
    r(   )r   r)   r*   r   listr-   r+   r   r	   r	   r   differentiable_all_gather5   s   r7   )typingr   r   r   torch.distributeddistributedr   torch.autograd.functionr   r   r'   r,   r-   r7   r	   r	   r	   r   <module>   s    