o
    rr©i"  ã                   @   sà  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	Z	d dl
mZ ddlmZ ddlmZmZmZ ddlmZmZ d	ed
dfdd„Zd	ed
dfdd„Zd	ed
dfdd„Zd	ed
dfdd„Zd	ed
dfdd„Zd	ed
dfdd„Zd	ed
dfdd„Zd	ed
dfdd„Ze	jjej dd¡ ddd	ed
dfdd „ƒZ d	ed
dfd!d"„Z!d	ed
dfd#d$„Z"d	ed
dfd%d&„Z#d	ed
dfd'd(„Z$d4d)e%d*e%d
e%fd+d,„Z&d	ed
dfd-d.„Z'd	ed
dfd/d0„Z(d1ed
dfd2d3„Z)dS )5é    N)ÚPath)ÚAutoExecutoré   )Úhelpers)Újob_environmentÚ	test_coreÚutilsé   )ÚlocalÚ
test_debugÚtmp_pathÚreturnc              	   C   sú  dt dt fdd„}t | ¡}|jddd | |d¡}|jdd | ¡ + tjtd	d
 |jdd W d   ƒ n1 s>w   Y  | |d¡}W d   ƒ n1 sSw   Y  | 	¡ g d¢ksbJ ‚| 
d¡ ¡ dksmJ ‚| 
d¡ ¡ dksxJ ‚| 
d¡ ¡ dksƒJ ‚| ¡ d u s‹J ‚| ¡ s‘J ‚tjtdd
 | 
d¡ ¡  W d   ƒ n1 sªw   Y  | 	¡ dgks¸J ‚| 
d¡ ¡ dksÃJ ‚| 
d¡|u sÌJ ‚| ¡ sÒJ ‚t |¡}t |¡}| 	¡ dgksåJ ‚|jd usìJ ‚~t |¡}|jd u sûJ dƒ‚d S )NÚpr   c                 S   s   t  ¡ }| |j S ©N)r   ÚJobEnvironmentÚ
local_rank)r   Újob_env© r   úM/home/ubuntu/.local/lib/python3.10/site-packages/submitit/local/test_local.pyÚfunc   s   
ztest_local_job.<locals>.funcé   r	   ©Útasks_per_nodeÚnodes)r   zwith executor.batch)Úmatchr   )r   r	   r   zmust be between 0 and 2é   r   z0garbage collection should I removed finished job)Úintr
   ÚLocalExecutorÚupdate_parametersÚsubmitÚbatchÚpytestÚraisesÚRuntimeErrorÚresultsÚtaskÚresultÚ	exceptionÚdoneÚ
ValueErrorÚpickleÚdumpsÚloadsÚ_process)r   r   ÚexecutorÚjob1Újob2ÚbÚjob3r   r   r   Útest_local_job   s>   

ÿýÿ


r3   c                 C   sP   t  d¡}t | ¡}| ||j|j¡}tt||j|jƒƒdd„ |D ƒks&J ‚d S )Né   c                 S   ó   g | ]}|  ¡ ‘qS r   ©r&   ©Ú.0Újr   r   r   Ú
<listcomp>E   ó    z(test_local_map_array.<locals>.<listcomp>)	r   ÚCheckFunctionr
   r   Ú	map_arrayÚdata1Údata2ÚlistÚmap)r   Úgr.   Újobsr   r   r   Útest_local_map_arrayA   s   

*rD   c                    sd   t  d¡‰ ‡ fdd„tˆ jˆ jƒD ƒ}t | ¡}| |¡}tt	ˆ ˆ jˆ jƒƒdd„ |D ƒks0J ‚d S )Nr4   c                    s   g | ]\}}t  ˆ ||¡‘qS r   )Ú	functoolsÚpartial)r8   ÚxÚy©rB   r   r   r:   J   s    z+test_local_submit_array.<locals>.<listcomp>c                 S   r5   r   r6   r7   r   r   r   r:   M   r;   )
r   r<   Úzipr>   r?   r
   r   Úsubmit_arrayr@   rA   )r   Úfnsr.   rC   r   rI   r   Útest_local_submit_arrayH   s
   


*rM   c                 C   s\   ddd„}t  | ¡}| |¡}| ¡ }t|tjƒsJ ‚|jd }d|v s&J ‚d|v s,J ‚d S )Nr   c                   S   s   t dƒ‚)NúFailed on purpose)r#   r   r   r   r   Úfailing_jobQ   s   z%test_local_error.<locals>.failing_jobr   Ú	TracebackrN   ©r   N)r
   r   r   r'   Ú
isinstancer   ÚFailedJobErrorÚargs)r   rO   r.   Újobr'   Ú	tracebackr   r   r   Útest_local_errorP   s   



rW   c                 C   s:   G dd„ dƒ}t  | ¡}| |j¡}t| ¡ |ƒsJ ‚d S )Nc                   @   s   e Zd ZdS )z-test_pickle_output_from_main.<locals>.MyClassN)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   ÚMyClass^   s    r[   )r
   r   r   Ú__call__rR   r&   )r   r[   r.   rU   r   r   r   Útest_pickle_output_from_main]   s   
r]   c                 C   sj   d
dd„}t  | ¡}|jddd | |¡}| ¡ }t|tjƒs"J ‚|jd }d|v s-J ‚d	|v s3J ‚d S )Nr   c                  S   s&   t  ¡ } | jdkrtd| j› ƒ‚d S )Nr   zFailed on purpose: )r   r   r   r#   ©r   r   r   r   Úflakyg   s   
ÿz(test_get_first_task_error.<locals>.flakyr   r	   r   r   rP   zFailed on purpose: 1rQ   )	r
   r   r   r   r'   rR   r   rS   rT   )r   r_   r.   rU   r'   rV   r   r   r   Útest_get_first_task_errorf   s   



r`   c                 C   s   ddd„}t  | ¡}|jddd | |¡}| ¡  | ¡ }|d us$J ‚d|v s*J ‚d|v s0J ‚| ¡ }|d us:J ‚d	|v s@J ‚d
|v sFJ ‚d S )Nr   c                  S   s*   t  ¡ } td| jƒ td| jtjd d S )Nz
hello fromzbye from)Úfile)r   r   Úprintr   ÚsysÚstderrr^   r   r   r   Úhellow   s   ztest_stdout.<locals>.hellor   r	   r   zhello from 0
zhello from 1
zbye from 0
zbye from 1
rQ   )r
   r   r   r   ÚwaitÚstdoutrd   )r   re   r.   rU   rg   rd   r   r   r   Útest_stdoutv   s   


rh   c                 C   sV   ddd„}t  | ¡}| |¡}|jdksJ ‚|j tj¡ t 	d¡ |jdks)J ‚d S )Nr   c                   S   s   t  d¡ tdƒ‚)Néx   rN   )ÚtimeÚsleepr#   r   r   r   r   rO      ó   
z test_killed.<locals>.failing_jobÚRUNNINGr	   ÚINTERRUPTEDrQ   )
r
   r   r   Ústater-   Úsend_signalÚsignalÚSIGKILLrj   rk   )r   rO   r.   rU   r   r   r   Útest_killedŒ   s   



rs   ÚSUBMITIT_SLOW_TESTSFÚslow)Úreasonc              	   C   sT   ddt dt dt dt fdd„}t | ¡}|jdd	 | |d
d¡}| ¡ dks(J ‚d S )Nri   rG   rH   rk   r   c                 S   s   t  |¡ | | S r   )rj   rk   )rG   rH   rk   r   r   r   Úf›   rl   z test_long_running_job.<locals>.fr4   ©Útimeout_miné(   r   é*   )ri   )r   r
   r   r   r   r&   )r   rw   r.   rU   r   r   r   Útest_long_running_job™   s
   
r|   c                 C   s¼   t jdd}tdƒD ]}|jtj|dd q
tj| dd}|jddd | 	|¡}| 
¡  | ¡ }|d us7J ‚t d	|¡}|sFJ d
|› ƒ‚d|v sQJ d|› ƒ‚d|v s\J d|› ƒ‚d S )NT)Úverboseé   r	   )rG   rk   ©Úmax_num_timeoutgš™™™™™©?©ry   Úsignal_delay_szStarting from [123]/20z'Should have resumed from a checkpoint:
ztimed-out too many timeszUnexpected stdout:
z(0 remaining timeouts))r   ÚFunctionSequenceÚrangeÚaddr   Ú
do_nothingr
   r   r   r   rf   rg   ÚreÚsearch)r   r   rG   r.   rU   rg   r   r   r   r   Útest_requeuing¥   s   
r‰   c                 C   sd   G dd„ dt jƒ}tj| dd}|jddd | |dƒ¡}| ¡  | ¡ }|s*J ‚d|v s0J ‚d S )	Nc                   @   s,   e Zd Zddefdd„Zddefdd„ZdS )	z'test_custom_checkpoint.<locals>.SlackerTÚslackc                 S   s2   |rt ddd t d¡ tdƒ‚t ddd dS )NÚSlackingT©Úflushé
   zI really don't want to workzWorking hardzworked hard)rb   rj   rk   r#   ©ÚselfrŠ   r   r   r   r\   ·   s   
z0test_custom_checkpoint.<locals>.Slacker.__call__c                 S   s   |rt ddd tj| ddS )Nz4Interrupted while slacking. I won't slack next time.TrŒ   F)rŠ   )rb   r   ÚDelayedSubmissionr   r   r   r   Ú__submitit_checkpoint__¿   s   z?test_custom_checkpoint.<locals>.Slacker.__submitit_checkpoint__N)T)rX   rY   rZ   Úboolr\   r’   r   r   r   r   ÚSlacker¶   s    r”   r	   r   g¡?r   TzI won't slack next time.)r   ÚCheckpointabler
   r   r   r   rf   rg   )r   r”   r.   rU   rg   r   r   r   Útest_custom_checkpointµ   s   r–   c                 C   sn   t j| ddd}t | t|jƒd¡}|j}| ¡  |j ¡ }|j	 ¡ }|r-t
| ¡ ƒ|ks5J d|› dƒ‚d S )Nz@python -c 'import os;print(os.environ["SUBMITIT_LOCAL_JOB_ID"])'r	   rx   r   zPID link is broken (stderr: ú))r
   Ústart_controllerr   ÚJobPathsÚstrÚpidrf   rg   Ú	read_textrd   r   Ústrip)r   ÚprocessÚpathsÚpgrg   rd   r   r   r   Útest_make_subprocessÍ   s   ÿ

(r¡   c                 C   s    t  | ¡}| tjd¡}|jdksJ ‚| ¡  t d¡ |jdks#J ‚| tjd¡}|j}| ¡  |jdks8J ‚| 	¡ d u s@J ‚~t d¡ | 	¡ dksNJ ‚d S )NrŽ   rm   gš™™™™™¹?rn   éþÿÿÿ)
r
   r   r   rj   rk   ro   Úcancelr-   Úcancel_at_deletionÚpoll)r   r.   rU   rž   r   r   r   Útest_cancelÙ   s   


r¦   rG   rH   c                 C   s   dS )NéB   r   )rG   rH   r   r   r   Úf66ì   s   r¨   c                 C   s^   t | dd}| d }|jd|› gd | td¡}t d¡ | ¡ dks'J ‚| ¡ s-J ‚d S )	Nr
   )ÚclusterÚ
setup_doneztouch )Úlocal_setupé   r	   r§   )r   r   r   r¨   rj   rk   r&   Úexists)r   r.   Ú
setup_filerU   r   r   r   Ú
test_setupð   s   
r¯   c                 C   sl   t  | ¡}|jtddd}t  | |j¡ ¡ }|jtu sJ ‚|jdks$J ‚|j	ddiks-J ‚|j
du s4J ‚dS )z9Check we can load submission just from a path and job id.éC   éD   )rH   )r°   rH   N)r
   r   r   r¨   ÚLocalJobÚjob_idÚ
submissionÚfunctionrT   ÚkwargsÚ_result)r   r.   rU   r´   r   r   r   Útest_load_submissionú   s   
r¸   Úweird_tmp_pathc                 C   s$   t  | d ¡}| tdd¡ ¡  d S )Nz%jr°   r±   )r
   r   r   r¨   r&   )r¹   r.   r   r   r   Útest_weird_dir  s   rº   )r   )*rE   Úosr*   r‡   rq   rc   rj   Úpathlibr   r!   Úsubmititr   Ú r   Úcorer   r   r   r
   r   r3   rD   rM   rW   r]   r`   rh   rs   ÚmarkÚskipifÚenvironÚgetr|   r‰   r–   r¡   r¦   r   r¨   r¯   r¸   rº   r   r   r   r   Ú<module>   s>   (	
