o
    }oi/                     @   s4   d dl Z d dlmZ d dlmZ G dd deZdS )    N)ContextDecoratorc                   @   s.   e Zd ZdZdd Zdd ZdefddZd	S )
FirstRankPerNodeuU  
    Context manager that:
      • Lets LOCAL_RANK==0 run the protected code first on each node.
      • Inserts an extra barrier across *only* the node‑local rank‑0 processes.
      • Works on a single GPU (no env flags, no distributed initialisation).

    Note: it is assumed the scoped code is not torch.distributed heavy.
    c                 C   sl   d| _ d | _d| _t s|  | _ t sdS tj}t }t	|
d|}|dk| _| js3t  | jS )NFT
LOCAL_RANKr   )_created_pg_node0_group_firstdistis_initialized_try_bootstrap_pgosenvironget_rankintgetbarrier)selfenvglobal_rank
local_rank r   M/home/ubuntu/.local/lib/python3.10/site-packages/nemo/automodel/dist_utils.py	__enter__   s   

zFirstRankPerNode.__enter__c                 C   sN   z| j rt rt  |d urt  W | jrt  dS | jr&t  w w )NF)r   r   r	   r   destroy_process_groupr   )r   exc_typeexc_valexc_tbr   r   r   __exit__?   s   
zFirstRankPerNode.__exit__returnc                    sL   t j d}t fdd|D r$tjdt dt dd dS d	S )
z1Try to create a default pg from env:// variables.)
WORLD_SIZERANKMASTER_ADDRMASTER_PORTc                 3   s    | ]}| v V  qd S )Nr   ).0kr   r   r   	<genexpr>Q   s    z5FirstRankPerNode._try_bootstrap_pg.<locals>.<genexpr>gloor   r   )backend
world_sizerankTF)r   r   allr   init_process_groupr   r   )r   requiredr   r$   r   r
   M   s   z"FirstRankPerNode._try_bootstrap_pgN)__name__
__module____qualname____doc__r   r   boolr
   r   r   r   r   r      s
    	 r   )r   
contextlibr   torch.distributeddistributedr   r   r   r   r   r   <module>   s   