o
    2wi                     @   sr   d Z ddlmZ ddlZddlZddlZddlZddl	m
Z
mZ ee
dZ	G dd dZdd	eje fd
dZdS )z)
Start multiple process locally for DDP.
    )partialN   )
simple_logfatalz	Executor:c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
ChildrenManagerc                 C   s   g | _ d| _d S )NF)childrenfailedself r   J/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/dora/executor.py__init__   s   
zChildrenManager.__init__c                 C   s   t | j|_| j| d S N)lenr   rankappend)r
   childr   r   r   add   s   zChildrenManager.addc                 C   s   | S r   r   r	   r   r   r   	__enter__   s   zChildrenManager.__enter__c              	   C   s   |d urt d|  d| _z<| jrI| jsIt| jD ](}z|d}W n
 tjy-   Y qw | j| |rBt d|j d d| _q| jrI| jrW n t	yZ   t d d| _Y nw | jD ]}|
  q^| jsnt d d S d S )Nz-An exception happened while starting workers Tg?zWorker z died, killing all workersz8Received keyboard interrupt, trying to kill all workers.z"All workers completed successfully)logr   r   listwaitspTimeoutExpiredremover   KeyboardInterrupt	terminate)r
   exc_type	exc_value	tracebackr   exitcoder   r   r   __exit__"   s6   


zChildrenManager.__exit__N)__name__
__module____qualname__r   r   r   r!   r   r   r   r   r      s
    r   num_workersc              
   C   s\  dd l }|p
|j }|std td | |}|jjddd |j	
 r-|j	  td| d t d}t|D ]W}i }ttj}	t||	d< t||	d	< d
|	d< ddd| jd| jddg}
|
|7 }
|dkrtj|d< t|jd| d d|d< tj|d< |tjtjg|
 fd|	i| q=W d    n1 sw   Y  tt|j d S )Nr   zODDP is only available on GPU. Make sure GPUs are properly configured with cuda.r   T)exist_okparentsz	Starting z worker processes for DDP.RANK
WORLD_SIZEz	127.0.0.1MASTER_ADDRz-mdoraz-Pz--main_modulerunz--stdinworker_z.logwstdoutstderrenv)torchcudadevice_countr   sysexitget_xpfoldermkdirrendezvous_fileexistsunlinkr   r   rangedictosenvironstrpackagemain_moduler   DEVNULLopenSTDOUTr   Popen
executableintr   )mainargvr%   th
world_sizexpmanagerr   kwargsr2   argsr   r   r   start_ddp_workers;   sB   






rS   r   )__doc__	functoolsr   r@   
subprocessr   r6   typingtpr   r   r   r   OptionalrJ   rS   r   r   r   r   <module>   s   
%