o
    wiM                  	   @   s|  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
Z
d dlZddlmZ ddlmZmZmZmZ ddlmZ dd	lmZ dsdeej dededdfddZe jdejej fddZdeddfddZdtddZdeddfddZe
j !ddde"deddfddZ#deddfddZ$e jdud!ej%e& d"e"fd#d$Z'dedej(fd%d&Z)deddfd'd(Z*deddfd)d*Z+deddfd+d,Z,dtd-d.Z-dtd/d0Z.dtd1d2Z/deddfd3d4Z0deddfd5d6Z1dtd7d8Z2e
j !d9g d:d;ed<eddfd=d>Z3e
j !d?d@dAgfdBdCgfdDdEgfdFg dGfdHdIgfgdJedKej4ej5ej6e&ef dLf  ddfdMdNZ7e
j !dOdPdQgdRedKeddfdSdTZ8dtdUdVZ9dtdWdXZ:dtdYdZZ;dtd[d\Z<e jd]edejej= fd^d_Z>dtd`daZ?dtdbdcZ@dtdddeZAdtdfdgZBdheddfdidjZCe
j !dki dldigdkejDee&f deddfdmdnZEdeddfdodpZFdeddfdqdrZGdS )v    N)Path)patch   )helpers)job_environment
submission	test_coreutils)Job   )slurm jobprintserrorsreturnc              	      sr    fdd j j j jfD }t|||fD ] \}}t|d}|| W d   n1 s1w   Y  qdS )zWrite fake log filesc                    s"   g | ]}t |d t  jqS )%j)strreplacejob_id.0xr    V/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/submitit/slurm/test_slurm.py
<listcomp>   s   " z#_mock_log_files.<locals>.<listcomp>wN)pathsstdoutstderrzipr   openwrite)r   r   r   	filepathsfilepathmsgfr   r   r   _mock_log_files   s   r(   c                  c   sr    t jdgd} z)|   | V  W d    n1 sw   Y  W tjj  d S W tjj  d S tjj  w )Nsrun)
known_cmds)r   MockedSubprocesscontextr   SlurmJobwatcherclearmockr   r   r   mocked_slurm    s   
r2   tmp_pathc                 C   s^   t  "}|dd t| d}|jdksJ |jdd W d    d S 1 s(w   Y  d S )N12z       UNKNOWNFtimeout)r2   set_job_stater   r-   state
_interrupt)r3   r1   r   r   r   r   test_mocked_missing_state+   s   "r;   c               	   C   s   t  5} | dd | d t jdksJ W d    n1 s#w   Y  W d    d S W d    d S 1 s;w   Y  d S )Nr4   RUNNINGr   )r2   r8   job_contextr   JobEnvironmentclusterr0   r   r   r   test_job_environment3   s   "r@   c              	   C   sJ  t  }tj| d}|jtjdddd}|jdksJ |jdks"J | d u s*J t	|dd	d
 d|_
ttj |  W d    n1 sIw   Y  t	|dd	d
 ||j t|jj W d    n1 smw   Y  | dkszJ | d	ksJ | dksJ dtjvsJ dW d    d S 1 sw   Y  d S )Nfolderr   r      blublur4   r<   This is the error log
hop)r   r   r      _USELESS_TEST_ENV_VAR_z(Test context manager seems to be failing)r2   r   SlurmExecutorsubmitr   
do_nothingr   r9   r   r(   _results_timeout_spytestraisesr	   UncompletedJobError_get_outcome_and_resultr=   r   process_jobr   rB   resultr    osenviron)r3   r1   executorr   r   r   r   test_slurm_job_mocked:   s(   
"rW   use_batch_api)FTc              
      s  d}t  }tj|d}|jdd t|tdd| dtdtdtffd	d
}g }| rY|  tD ]\}}||	||| q:W d    n1 sSw   Y  n|
|}|d jdd   fddt|D dd |D ks~J |D ]&}	|	jdksJ ||	j t|	jj W d    n1 sw   Y  q|d jdd tt|dd |D ksJ t| djj }
dd |
 D }|dgksJ W d    d S 1 sw   Y  d S )N   rA   rC   )array_parallelism
   r   yr   c                    s    |  v sJ |v sJ | | S Nr   )r   r\   )data1data2r   r   addY   s   z(test_slurm_job_array_mocked.<locals>.addr   _c                    s   g | ]	}  d | qS )ra   r   )r   a)array_idr   r   r   f   s    z/test_slurm_job_array_mocked.<locals>.<listcomp>c                 S   s   g | ]}|j qS r   r   r   jr   r   r   r   f   s    r<   Tr6   c                 S   s   g | ]}|  qS r   )rS   re   r   r   r   r   n   s    rd   c                 S   s   g | ]
}d |v r|  qS )z--array)strip)r   lr   r   r   r   q   s    z#SBATCH --array=0-4%3)r2   r   rJ   update_parametersrangeintbatchr!   appendrK   	map_arrayr   splitr9   r=   r   rR   r   rB   r:   listmapr
   submission_file	read_text
splitlines)rX   r3   nr1   rV   r`   jobsd1d2r   sbatch
array_liner   )rc   r^   r_   r   test_slurm_job_array_mockedQ   s:   
(""r{   c              
   C   s  t  ~}tj| d}|jddd |jtjdddd}||j$ t	
t t|jj W d    n1 s9w   Y  W d    n1 sHw   Y  t|d	d
 t	
tj |  W d    n1 shw   Y  | }t|tjsyJ W d    d S 1 sw   Y  d S )NrA      r   )timegpus_per_noder   r   rH   )errorrF   )r   )r2   r   rJ   ri   rK   r   rL   r=   r   rN   rO   
ValueErrorr   rR   r   rB   r(   r	   FailedJobErrorrS   	exception
isinstance)r3   r1   rV   r   r   r   r   r   test_slurm_error_mockedu   s"   
"r   Fcalled_with
not_calledc              
   c   s    |s	| d us	J t dd d}|0}zd V  W |r|  n||  n|r+|  w ||  w W d    d S W d    d S 1 sDw   Y  d S )Nz1submitit.slurm.slurm.SlurmJobEnvironment._requeue)return_value)r   assert_not_calledassert_called_with)r   r   requeue_patchr   r   r   mock_requeue   s   

"r   c                 C   s,   t  }tj| jj}t|| j|}|S r]   )	r   SlurmJobEnvironmentr	   DelayedSubmissionloadr   submitted_pickler   SignalHandler)r   envdelayedsigr   r   r   get_signal_handler   s   r   c              	   C   s  t j }t }|tjd t|tj	sJ t
  tj| dd}|jdd ||}W d    n1 s8w   Y  t|}|dd tt" tdd || W d    n1 saw   Y  W d    n1 spw   Y  t|}|d	d tt" td
d || W d    n1 sw   Y  W d    n1 sw   Y  t|}|dd tj }tdd. tjtjdd || W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr[   r   rB   max_num_timeout<   r}      minutesr   2   r   7   Tr   ztimed-out too many times.match)submititr>   _usr_sigr   FunctionSequencer`   r   _three_timer   Checkpointabler2   r   rJ   ri   rK   r   rN   rO   
SystemExitr   checkpoint_and_try_requeuer   r	   rP   )r3   fast_forward_clockusr_sigfs0rV   r   r   r   r   r   test_requeuing_checkpointable   s>   

 
 

Pr   c              	   C   s  t j }t  tj| dd}|jdd |tj	d}W d    n1 s'w   Y  t
|}|dd tdd	 |tjj W d    n1 sLw   Y  tt" tdd
 || W d    n1 slw   Y  W d    n1 s{w   Y  t
|}|dd tdd	. tjtjdd || W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   r   r   r   r[   r   r   Tr   r   r   z timed-out and not checkpointabler   )r   r>   r   r2   r   rJ   ri   rK   r   r   r   r   bypasssignalSignalsSIGTERMrN   rO   r   r   r	   rP   )r3   r   r   rV   r   r   r   r   r   !test_requeuing_not_checkpointable   s.   

 
Pr   c              	   C   s   t j }t  tj| dd}|jdd |tj	d}W d    n1 s'w   Y  t
|}tt" tdd || W d    n1 sKw   Y  W d    n1 sZw   Y  tj|jj}|jdksnJ d S )Nr   r   r   r   r[   Tr   )r   r>   r   r2   r   rJ   ri   rK   r   r   r   rN   rO   r   r   checkpoint_and_exitr	   r   r   r   r   _timeout_countdown)r3   r   rV   r   r   r   r   r   r   test_checkpoint_and_exit   s   
 r   c               
   C   s  t jddddddig dd} d	| v sJ d
| vsJ d| vs!J ttjd }| s1||  | }g }tt	| 
 |
 D ]\}\}}||kr]|d|d  d| d| d qB|rt|  td t| dg| }|ddd| g7 }td|d S )Nz
blublu bar/tmp	learnfairTrE   rH   )z-vvz
--cpu-bindnone)commandrB   	partition	exclusiveadditional_parameters	srun_argsr   z	--command
constraintz_sbatch_test_record.txtzline #r   z: "z" -> ""z	# # # # #zDifference with reference file:r   z)Delete the record file if this is normal:zrm 
)r   _make_sbatch_stringr   __file__parentexists
write_textrs   	enumerater!   rt   rm   printAssertionErrorjoin)stringrecord_filerecordedchangeskline1line2messager   r   r   test_make_sbatch_string   s8   
""
r   c                  C   s    t jdddd} d| v sJ d S )NrE   r   r   )r   rB   r~   z--gpus-per-node=2r   r   r   r   r   r   test_make_sbatch_string_gpu     r   c                  C   s    t jdddd} d| vsJ d S )NrE   r   T)r   rB   stderr_to_stdout--errorr   r   r   r   r   test_make_sbatch_stderr  r   r   c                 C   sV   t   tj| d}W d    n1 sw   Y  |jdd |jjd dks)J d S )NrA   g      @)mem_gbmem3584MB)r2   r   AutoExecutorri   	_executor
parametersr3   rV   r   r   r   test_update_parameters  s
   r   c                 C   sn   t   tj| d}W d    n1 sw   Y  tt |jdd W d    d S 1 s0w   Y  d S )NrA   rH   rD   )r2   r   rJ   rN   rO   r   ri   r   r   r   r   test_update_parameters_error  s   "r   c                  C   sV   d} t  | }|d dddksJ |d dddksJ t|h dks)J d S )	NzoJobID|State
5610980|RUNNING
5610980.ext+|RUNNING
5610980.0|RUNING
20956421_0|RUNNING
20956421_[2-4%25]|PENDING
5610980r<   )JobIDState
20956421_2z20956421_[2-4%25]PENDING>   r   
20956421_0r   
20956421_3
20956421_4)r   SlurmInfoWatcher	read_infoset)exampleoutputr   r   r   test_read_info&  s
   r   z
name,state))12_0R)12_1U)12_2X)12_3r   )12_4r   namer9   c                 C   sJ   d}t  }dD ]}|| q||}|| i dd|ks#J d S )Nz JobID|State
12_0|R
12_[2,4-12]|X)r   r   r   r   )r   r   register_jobr   get)r   r9   r   r.   jobidr   r   r   r   test_read_info_array4  s   
 r   zjob_id,expectedr4   )rH   r   )rH   r   z20_[2-7%56]   r      z20_[2-7,12-17,22%56])r   )r   rH      )r      z20_[0%1])r   r   r   expected.c                 C   s$   t | }|dd |D ksJ d S )Nc                 S   s   g | ]}t d d |D qS )c                 s   s    | ]}t |V  qd S r]   )r   r   r   r   r   	<genexpr>L  s    z.test_read_job_id.<locals>.<listcomp>.<genexpr>)tuple)r   groupr   r   r   r   L  s    z$test_read_job_id.<locals>.<listcomp>)r   read_job_id)r   r  r   r   r   r   test_read_job_id@  s   
r  zstring,expected)s   Submitted batch job 5610208
5610208)zSubmitted batch job 5610208
r	  r   c                 C   s   t j| }||ksJ d S r]   )r   rJ   #_get_job_id_from_submission_command)r   r  r   r   r   r   #test_get_id_from_submission_commandO  s   r  c                   C   s@   t tj tjjdd W d    d S 1 sw   Y  d S )Ns   blublur   )rN   rO   r	   FailedSubmissionErrorr   rJ   r
  r   r   r   r   )test_get_id_from_submission_command_raiseX  s   "r  c                  C   s   t  ^} t }| dd |jdksJ |jdd}t|j dhks'J |j	dhks/J |dks5J | dd |jddd	}|dksHJ |j	ddhksQJ |j
dhksYJ W d    d S 1 sdw   Y  d S )
Nr4   r<   r   11rd   r5   FAILEDforce)r   mode)r2   r   r   r8   	num_calls	get_stater   
_info_dictkeys_registered	_finished)r1   r.   r9   r   r   r   test_watcher]  s   "r  c                  C   s   t  } | d dksJ d S )Nnodesr   )r   _get_default_parameters)defaultsr   r   r   test_get_default_parameterso  s   r  c                   C   s   t j dks	J d S )Nr   )r   rJ   r   r   r   r   r   	test_namet  s   r  	node_listc                 c   s4    dt jd< | t jd< t V  t jd= t jd= d S )N1SLURM_JOB_IDSLURM_JOB_NODELIST)rT   rU   r   r   )r  r   r   r   with_slurm_job_nodelistx  s   


r"  c                  C   sL  t d} dg| jksJ W d    n1 sw   Y  t d} ddg| jks*J W d    n1 s4w   Y  t d} ddg| jksGJ W d    n1 sQw   Y  t d} d	d
g| jksdJ W d    n1 snw   Y  t d} g d| jksJ W d    n1 sw   Y  t d} g d| jksJ W d    n1 sw   Y  t d} g d| jksJ W d    n1 sw   Y  t d} g d| jksJ W d    n1 sw   Y  t d} ddg| jksJ W d    n1 sw   Y  t d} g d| jksJ W d    d S 1 sw   Y  d S )Nzcompute-b24zcompute-a1,compute-b2
compute-a1z
compute-b2zcompute-b2[1,2]compute-b21compute-b22zcompute-b2[011,022]zcompute-b2011zcompute-b2022zcompute-b2[1-3])r$  r%  compute-b23zcompute-b2[1-3,5,6,8])r$  r%  r&  compute-b25compute-b26compute-b28zcompute-b2[1-3,5-6,8]z compute-b2[1-3,5-6,8],compute-a1)r$  r%  r&  r'  r(  r)  r#  zcompute[042,044]
compute042
compute044zcompute[042-043,045,048-049])r*  
compute043
compute045
compute048
compute049r"  	hostnamesr   r   r   r   test_slurm_node_list  sH   







	

	



$r3  c                  C   s@   t d} g d| jksJ W d    d S 1 sw   Y  d S )Nz)compute-b24-[1-3,5-9],compute-b25-[1,4,8])zcompute-b24-1zcompute-b24-2zcompute-b24-3zcompute-b24-5zcompute-b24-6zcompute-b24-7zcompute-b24-8zcompute-b24-9zcompute-b25-1zcompute-b25-4zcompute-b25-8r0  r2  r   r   r   )test_slurm_node_list_online_documentation  s
   

"r4  c               	   C   s   t tj! td} t| j W d    n1 sw   Y  W d    n1 s*w   Y  t tj* td} t| j W d    n1 sJw   Y  W d    d S W d    d S 1 sbw   Y  d S )Nzcompute-b2[1-,4]zcompute-b2[1,2,compute-b3])rN   rO   r   SlurmParseExceptionr"  r   r1  r2  r   r   r   test_slurm_invalid_parse  s   

"r6  c                  C   s@   t d} | jg| jksJ W d    d S 1 sw   Y  d S )Nr   )r"  hostnamer1  r2  r   r   r   test_slurm_missing_node_list  s   
"r8  weird_tmp_pathc                 C   s  d| j v r
td t  tj| d}|jtjdddd}W d    n1 s(w   Y  |j	j
d |j	jd i }|j	j  D ]*}|d	sMqEd
|vrRqE|td	d   d
d\}}|d|jdd||< qEtjd|d  dd tjd|d  dd d S )Nr   z0test doesn't support newline in 'weird_tmp_path'rA   r   r   rC   rD   r   z#SBATCH=r   z%t0zls z--outputT)shellr   )r   rN   skipr2   r   rJ   rK   r   rL   r   r   r   r    rr   rs   rt   
startswithlenrg   ro   r   r   
subprocess
check_call)r9  rV   r   sbatch_argsrh   keyvalr   r   r   test_slurm_weird_dir  s$   


 rE  paramsr   c                 C   s   t  & tj|d}|jdi | dddii |jtjdddd}W d    n1 s,w   Y  |jj	 }d	d
 |
 D }t|dksMJ d| d S )NrA   slurm_additional_parametersmem_per_gpurH   r   r   rC   rD   c                 S   s   g | ]}d |v r|qS )z#SBATCH --memr   r   r   r   r   r     s    z+test_slurm_through_auto.<locals>.<listcomp>zUnexpected lines: r   )r2   r   r   ri   rK   r   rL   r   rr   rs   rt   r?  )rF  r3   rV   r   text	mem_linesr   r   r   test_slurm_through_auto  s   rK  c              	   C   s   dd }t  k}tj| d}||}t|dd d|_tjtj	dd |
  W d    n1 s3w   Y  |d	d
 ||}d|_tjtj	dd |
  W d    n1 s]w   Y  W d    d S W d    d S 1 suw   Y  d S )Nc                   S   s   t d)NzToo bad)r   r   r   r   r   fail_silently  s   z/test_slurm_job_no_stderr.<locals>.fail_silentlyrA   zjob is running ...
)r   r   zjob is running ...r   13r<   z!No output/error stream produced !)r2   r   rJ   rK   r(   rM   rN   rO   r	   rP   rQ   r8   )r3   rL  r1   rV   r   r   r   r   test_slurm_job_no_stderr  s$   



"rN  c                 C   s   t  $ tj| d}|jddidgdgd |jtjddd	d
}W d    n1 s*w   Y  |jj	 }d|v s;J d|v sAJ d S )NrA   rH  rH   zThis is a setup commandzThis is a teardown command)rG  slurm_setupslurm_teardownr   r   rC   rD   )
r2   r   r   ri   rK   r   rL   r   rr   rs   )r3   rV   r   rI  r   r   r   /test_slurm_through_auto_with_setup_and_teardown
  s   rQ  )r   r   )r   N)NF)H
contextlibrT   r   r@  typingtppathlibr   unittest.mockr   rN   r   r   r   corer   r   r   r	   	core.corer
   r   Anyr   r(   contextmanagerIteratorr+   r2   r;   r@   rW   markparametrizeboolr{   r   Optionalrk   r   r   r   r   r   r   r   r   r   r   r   r   r   ListTupleUnionr  r  r  r  r  r  r   r"  r3  r4  r6  r8  rE  DictrK  rN  rQ  r   r   r   r   <module>   s   "

#*



	
2






-

	"
