o
    ٷiB                     @   sH   d dl Z d dlmZ dd Zdd Zdd Zdd	 Zd
d Zdd Z	dS )    Nc                  C   s   dt jv r%tt jd  tt jd } tt jd }tjdd|| d d S dt jv rPtt jdd tt jd	d} tt jd
d}tjdd|| d d S 	 d S )N
LOCAL_RANKRANK
WORLD_SIZEncclztcp://127.0.0.1:7645)init_method
world_sizerankOMPI_COMM_WORLD_LOCAL_RANK0OMPI_COMM_WORLD_RANKOMPI_COMM_WORLD_SIZE1ztcp://127.0.0.1:7647)osenvironintdistinit_process_groupget)r   r    r   g/home/ubuntu/.local/lib/python3.10/site-packages/onnxruntime/transformers/models/llama/dist_settings.py	init_dist   s   

r   c                  C   s.   zddl m}  | j}|W S  ty   Y d S w )Nr   )MPI)mpi4pyr   
COMM_WORLDImportError)r   commr   r   r   	_get_comm   s   r   c                  C      t  } | d ur|  S dS Nr   )r   Get_rankr   r   r   r   get_rank'      r!   c                  C   r   )N   )r   Get_sizer    r   r   r   get_size,   r"   r%   c                  C   s   t  } | d ur|   d S d S )N)r   Barrierr    r   r   r   barrier1   s   r'   c                  G   s   t  dkrt|   d S d S r   )r!   print)argsr   r   r   	print_out7   s   
r*   )
r   torch.distributeddistributedr   r   r   r!   r%   r'   r*   r   r   r   r   <module>   s   
