o
    wOi/                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
mZmZ d dlmZmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd deZG dd	 d	eZdS )
    N)datetime)DictIterableListOptionalTuple)
record_tsmTsmEvent)AppDryRunInfo	AppHandleApplication	AppStatus	RunConfig	SchedulerSchedulerBackendSessionSessionMismatchExceptionUnknownAppExceptionmake_app_handleparse_app_handlerunoptsc                       s  e Zd Zdef fddZdedefddZdedee	 fd	d
Z
dedee	 fddZdeeef fddZdeddfddZdedee fddZ				d0dedededee dee dee defddZejdedefddZejdedee	 fddZejdedee	 fd d!Zejdeeef fd"d#Zejdeddfd$d%Zejdedee fd&d'Zej				d0dedededee dee dee defd(d)Z		d1d*ed+ed,ee d-ee def
d.d/Z  Z S )2LoggingSessionnamec                    s   t  | d S N)super__init__)selfr   	__class__ ^/home/ubuntu/.local/lib/python3.10/site-packages/torchelastic/tsm/driver/standalone_session.pyr   %   s   zLoggingSession.__init__dryrun_inforeturnc                 C   s~   |j }|jrt|jjnd }| jd||d}z| |}t|\}}}||_t	| |W S  t
y>   t |_t	|  w )Nschedule)runcfg)
_scheduler_cfgjsondumpscfgs_generate_tsm_event	_scheduler   app_idr   	Exception	traceback
format_excraw_exception)r   r!   scheduler_backendr$   	tsm_event
app_handle_r,   r   r   r    r#   +   s$   

zLoggingSession.scheduler3   c                 C   X   t |\}}}| d||}z| |}t| |W S  ty+   t |_t|  w )Nstatus)r   r*   _statusr   r-   r.   r/   r0   )r   r3   r1   r4   r,   r2   
app_statusr   r   r    r6   A   s   

zLoggingSession.statusc                 C   sT   t |\}}}| d||}z
t| | |W S  ty)   t |_t|  w )Nwait)r   r*   r   _waitr-   r.   r/   r0   r   r3   r1   r4   r,   r2   r   r   r    r9   N   s   
zLoggingSession.waitc                 C   sF   |  dd}z|  }t| |W S  ty"   t |_t|  w )Nlist )r*   _listr   r-   r.   r/   r0   )r   r2   resr   r   r    r<   Y   s   
zLoggingSession.listNc                 C   sX   t |\}}}| d||}z| | t| W d S  ty+   t |_t|  w )Nstop)r   r*   _stopr   r-   r.   r/   r0   r;   r   r   r    r@   d   s   

zLoggingSession.stopc                 C   r5   )Ndescribe)r   r*   	_describer   r-   r.   r/   r0   )r   r3   r1   r4   r,   r2   r?   r   r   r    rB   s   s   

zLoggingSession.describer   	role_namekregexsinceuntilc                 C   sb   t |\}}}	| d||	}
z| ||||||}t|
 |W S  ty0   t |
_t|
  w )N	log_lines)r   r*   
_log_linesr   r-   r.   r/   r0   )r   r3   rD   rE   rF   rG   rH   r1   r4   r,   r2   log_iterr   r   r    rI      s   	
zLoggingSession.log_linesc                 C      t  r   NotImplementedError)r   r!   r   r   r    r+         zLoggingSession._schedulec                 C   rL   r   rM   r   r3   r   r   r    r7      rO   zLoggingSession._statusc                 C   rL   r   rM   rP   r   r   r    r:      rO   zLoggingSession._waitc                 C   rL   r   rM   r   r   r   r    r>      rO   zLoggingSession._listc                 C   rL   r   rM   rP   r   r   r    rA      rO   zLoggingSession._stopc                 C   rL   r   rM   rP   r   r   r    rC      rO   zLoggingSession._describec                 C   rL   r   rM   )r   r3   rD   rE   rF   rG   rH   r   r   r    rJ      s   
zLoggingSession._log_linesapi	schedulerr,   r$   c              	   C   s(   t |  ||t tt ||dS )N)sessionrS   rR   	unix_usersource_hostnamer,   r$   )r	   r   getpassgetusersocketgetfqdngethostname)r   rR   rS   r,   r$   r   r   r    r*      s   z"LoggingSession._generate_tsm_eventr   NNN)NN)!__name__
__module____qualname__strr   r
   r   r#   r   r   r6   r9   r   r   r<   r@   rB   intr   r   rI   abcabstractmethodr+   r7   r:   r>   rA   rC   rJ   r	   r*   __classcell__r   r   r   r    r   $   s    
r   c                       sf  e Zd Z	d.dedeeef def fddZdedefd	d
Z		d/de
dedeeeef fddZdede
fddZdedededefddZdeeef fddZdee fddZde
dee fddZde
dee fddZdee
ef fdd Zde
dd!fd"d#Zde
dee fd$d%Z	&	!	!	!d0de
d'ed(ed)ee d*ee d+ee defd,d-Z   Z!S )1StandaloneSession
   r   
schedulerswait_intervalc                    s<   d|vrt d|  t | || _|| _i | _d S )Ndefaultz6A default scheduler is required. Provided schedulers: )
ValueErrorkeysr   r   _schedulers_wait_interval_apps)r   r   rg   rh   r   r   r    r      s   
zStandaloneSession.__init__rS   r"   c                 C   s.   | j |}|std| d| j   |S )NzUndefined scheduler backend: z. Use one of: )rl   getKeyErrorrk   )r   rS   schedr   r   r    r%      s   zStandaloneSession._schedulerTr3   check_sessionc                 C   s<   t |\}}}|r| j|krt|| j| |}|||fS )a  
        Returns the scheduler and app_id from the app_handle.
        Set ``check_session`` to validate that the session name in the app handle
        is the same as this session.

        Raises:
            ValueError - if ``check_session=True`` and the session in the app handle
                         does not match this session's name
            KeyError - if no such scheduler backend exists
        )r   _namer   r%   )r   r3   rr   r1   session_namer,   rS   r   r   r    _scheduler_app_id   s
   

z#StandaloneSession._scheduler_app_idr!   c                 C   s8   |j }|  |}||}t|| j|}|j| j|< |S r   )r%   r#   r   rs   _apprn   )r   r!   r1   rq   r,   r3   r   r   r    r+      s   

zStandaloneSession._scheduleappcfgc                 C   s"   |  |}||| |||S r   )r%   	_validatesubmit_dryrun)r   rw   rS   rx   rq   r   r   r    _dryrun  s   
zStandaloneSession._dryrunc                 C   s   dd | j  D S )Nc                 S   s   i | ]	\}}||  qS r   )run_opts).0r1   rS   r   r   r    
<dictcomp>  s    z.StandaloneSession.run_opts.<locals>.<dictcomp>)rl   itemsrQ   r   r   r    r|     s   zStandaloneSession.run_optsc                 C   s   t | j S r   )r<   rl   rk   rQ   r   r   r    scheduler_backends  s   z$StandaloneSession.scheduler_backendsc                 C   s`   | j |dd\}}}||}|s| j|d  d S t|j|j|j|j|j	d}|r.|j
|_
|S )NFrr   )replicas)ru   rB   rn   popr   statenum_restartsmsgstructured_error_msgr   ui_url)r   r3   rS   r1   r,   descr8   r   r   r    r7     s"   
zStandaloneSession._statusc                 C   sB   | j |dd\}}}	 | |}|sd S | r|S t| j qNFr   )ru   r6   is_terminaltimesleeprm   )r   r3   rS   r1   r,   r8   r   r   r    r:   3  s   
zStandaloneSession._waitc                 C   s(   t | j }|D ]}| | q	| jS r   )r<   rn   rk   r6   )r   app_idsr,   r   r   r    r>   A  s   zStandaloneSession._listNc                 C   s<   |  |\}}}| |}|d u s| rd S || d S r   )ru   r6   r   cancel)r   r3   rS   r1   r,   r6   r   r   r    rA   I  s
   
zStandaloneSession._stopc                 C   sN   | j |dd\}}}| j|d }|r|S ||}|sd S t|dj|j S )NFr   )r   )ru   rn   ro   rB   r   ofroles)r   r3   rS   r1   r,   rw   r   r   r   r    rC   Q  s   
zStandaloneSession._describer   rD   rE   rF   rG   rH   c                 C   s>   |  |s	t|| j|dd\}}}	||	|||||}
|
S r   )r6   r   ru   rK   )r   r3   rD   rE   rF   rG   rH   rS   r1   r,   rK   r   r   r    rJ   a  s   
	zStandaloneSession._log_lines)rf   )Tr\   )"r]   r^   r_   r`   r   r   r   ra   r   r%   r   boolr   ru   r
   r+   r   r   r{   r   r|   r   r   r   r   r7   r:   r>   rA   rC   r   r   rJ   rd   r   r   r   r    re      sl    
	


re   ) rb   rW   r'   rY   r   r.   r   typingr   r   r   r   r   torchelastic.eventsr   r	   torchelastic.tsm.driver.apir
   r   r   r   r   r   r   r   r   r   r   r   r   r   re   r   r   r   r    <module>   s   < /