o
    `۷i                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ G dd deZG d	d
 d
e
jZdd Z					 	ddededededededededefddZdS )    N)Counter)tune)safe_write_to_results_json)
Checkpoint)Callbackc                   @   s   e Zd Zdd Zdd ZdS )ProgressCallbackc                 C   s   d| _ d| _d S )Nr   <   )last_updateupdate_intervalself r   V/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/tune/utils/release_test_util.py__init__   s   
zProgressCallback.__init__c                 K   sR   t   | j | jkr't   }||ttdd |D d}t|d || _d S d S )Nc                 S      g | ]}|j qS r   status.0trialr   r   r   
<listcomp>       z0ProgressCallback.on_step_end.<locals>.<listcomp>)r	   	iterationtrial_statesz/tmp/release_test_out.json)timer	   r
   dictr   r   )r   r   trialskwargsnowresultr   r   r   on_step_end   s   

zProgressCallback.on_step_endN)__name__
__module____qualname__r   r    r   r   r   r   r      s    r   c                       sD   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Z  Z	S )TestDurableTrainablec                    s"   |    tt| j|i | d S N)	setup_envsuperr$   r   )r   argsr   	__class__r   r   r   #   s   zTestDurableTrainable.__init__c                 C      d S r%   r   r   r   r   r   r&   (      zTestDurableTrainable.setup_envc                 C   sL   t |d | _|d | _|d | _|d | _|d | _| jd | _d| _d S )N	num_iters
sleep_timescorecheckpoint_iterscheckpoint_size_b   r   )int
_num_iters_sleep_time_score_checkpoint_iters_checkpoint_size_b_checkpoint_num_items_iter)r   configr   r   r   setup+   s   




zTestDurableTrainable.setupc                 C   sN   | j dkrt| j t| j | j d}| j | jkrd|d< |  j d7  _ |S )Nr   )r/   Tdone   )r:   r   sleepr5   r   r6   r4   )r   resr   r   r   step6   s   
zTestDurableTrainable.stepc                 C   s^   t j|d}tjjdd| jd}t|d}t	|| W d    d S 1 s(w   Y  d S )Nz
bogus.ckptr   r>   sizewb)
ospathjoinnprandomuniformr9   openpickledump)r   tmp_checkpoint_dircheckpoint_filecheckpoint_datafpr   r   r   save_checkpointB   s
   "z$TestDurableTrainable.save_checkpointc                 C   r+   r%   r   )r   
checkpointr   r   r   load_checkpointH   r,   z$TestDurableTrainable.load_checkpoint)
r!   r"   r#   r   r&   r<   rA   rR   rT   __classcell__r   r   r)   r   r$   "   s    r$   c                 C   s4  t | d }| d }| d }| d }| d }|d }| d }t|D ]u}d|| i}	|dkr|dkr|| dkrt H}
t|D ]1}tj|
d	| d
}tjj	dd|d}t
|d}t|| W d    n1 smw   Y  qAtj|	t|
d W d    n1 sw   Y  nt|	 t| q"d S )Nr-   r.   r/   r0   r1   r2   checkpoint_num_filesr   bogus_z.ckptr>   rB   rD   )rS   )r3   rangetempfileTemporaryDirectoryrE   rF   rG   rH   rI   rJ   rK   rL   rM   r   reportr   from_directoryr   r?   )r;   r-   r.   r/   r0   r1   checkpoint_num_itemsrV   imetricstmpdirrO   rP   rQ   r   r   r   function_trainableL   s4   

ra   r>   ,  namenum_samplesresults_per_secondtrial_length_smax_runtimecheckpoint_freq_sr1   rV   returnc                 K   s  d|v o|d o|d  dp|d  d}	d| }
t||
 }d}|dkr,t||
 }tdd||
|||d}td	|  d
dd}|| t}|	rTt}||d< t	 }tj
|f||dd|}t	 | }|ttdd |jD t d}tjdd}t|d}t|| W d    n1 sw   Y  ||k}|std|  d|dd|dd|   d|dd|dd |S td|  d|dd|dd|   d|dd |dd |S )!Nstorage_pathzs3://zgs://g      ?rc   r   g        )r/   r-   r.   r0   r1   rV   z Starting benchmark with config: T   )reuse_actorsverbosecheckpoint_freqF)r;   re   raise_on_failed_trialc                 S   r   r   r   r   r   r   r   r      r   z"timed_tune_run.<locals>.<listcomp>)
time_takenr   r	   TEST_OUTPUT_JSONz/tmp/tune_test.jsonwtzThe z test took z.2fz' seconds, but should not have exceeded z% seconds. Test failed. 

--- FAILED: z ::: z > z ---z' seconds, which is below the budget of z) seconds. Test successful. 

--- PASSED: z <= )
startswithr3   r   rJ   printupdatera   r$   r   	monotonicrunr   r   r   rE   environgetrK   jsonrM   upper)rd   re   rf   rg   rh   ri   r1   rV   tune_kwargsdurabler.   r-   r0   r;   
run_kwargs_train
start_timeanalysisrq   r   test_output_jsonfsuccessr   r   r   timed_tune_runj   s   	
	

r   )r>   r>   rb   rc   r   r>   )r{   rE   rL   rY   r   collectionsr   numpyrH   rayr   ray._private.test_utilsr   ray.tuner   ray.tune.callbackr   r   	Trainabler$   ra   strr3   boolr   r   r   r   r   <module>   sN    *!
