o
    `۷i<D                  3   @   sb  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
mZmZ d dlZd dlm  mZ d dlmZmZ d dlmZmZ d dlmZmZmZ d dlmZ d dlmZ d d	lm Z  d d
l!m"Z"m#Z# d dl$m%Z% 			ddde
e& de'de
e& dee'e&f de#f
ddZ(de
e& de
e	e&ef  fddZ)dd Z*dd Z+de#de&de"fddZ,de#de&de"fdd Z-e.d!d"d# Z/e/0 ej1d$e&ddd%d&ej1d'e&ddd(d&ej1d)e&ddd*d&ej1d+e&ddd,d&ej1d-e&ddd.d&ej1d/e&ddd0d&ej1d1e&ddd2d&ej1d3de2d4d5ej1d6de2d7d5ej1d8de3d9d5ej1d:de&d;d5ej1d<de&d=d5ej1d>de'dd?d@eeej4dAdBdej5dCe%de
e& de
e& dDe
e& dEe
e& dFe
e& dGe
e& dHe
e& dAee& dIe
ee3e2f  dJe
ee3e2f  dKe
e3 dLe
e& dMe
e& dNe'dee'e&f de
e& f dOdPZ6e/0 ej1d$e&ddd%d&ej4dQe&dReee%dSdTde
e& de&de
e& dee'e&f fdUdVZ7e/0 ej1d$e&ddd%d&ej1d>de'ddWd@ej4dQe&dReee%dSdTde
e& dNe'de&de
e& dee'e&f f
dXdYZ8e/0 ej1d$e&ddd%d&ej4dQe&dReee%dSdTde
e& de&de
e& dee'e&f fdZd[Z9e/0 ej1d$e&ddd%d&ej4dQe&dRej1d\d]de'dd^d@eee%dSdTde
e& de&d_e'de
e& dee'e&f f
d`daZ:e/0 ej1d$e&ddd%d&eee%dSdTde
e& de
e& dee'e&f fdbdcZ;dS )e    N)list2cmdline)AnyDictOptionalTupleUnion)get_or_create_event_loop
load_class)parse_metadata_jsonparse_resources_json)add_click_logging_optionscf
cli_logger)parse_runtime_env_args)add_common_job_options)redact_url_password)	JobStatusJobSubmissionClient)	PublicAPIFTaddresscreate_cluster_if_neededheadersverifyreturnc                 C   s0   t | |t||d}| }tdt| |S )Nr   r   zJob submission server address)r   _handle_headersget_addressr   labeled_valuer   )r   r   r   r   clientclient_address r    S/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/dashboard/modules/job/cli.py_get_sdk_client   s   r"   c              
   C   s^   | d u rdt jv rt jd } | d ur-zt| W S  ty, } ztd| |d }~ww d S )NRAY_JOB_HEADERSzbFailed to parse headers into JSON.
                Expected format: {{"KEY": "VALUE"}}, got {}, {})osenvironjsonloads	Exception
ValueErrorformat)r   excr    r    r!   r   /   s   
r   c                 C   B   t   t dt|   t |  t dt|   t   d S N-)r   newlinesuccesslensuccess_msgr    r    r!   _log_big_success_msg?   
   
r4   c                 C   r,   r-   )r   r/   errorr1   r2   r    r    r!   _log_big_error_msgG   r5   r7   r   job_idc                 C   s   |  |}|jtjkrtd| d |jS |jtjkr(td| d |jS |jtjkrIt	d| d |j
d urFtjd|j
 dd |jS td| d	|j  |j
d uretjd|j
 dd |jS )
NJob 'z' succeededz' was stoppedz' failedzStatus message: T	no_formatzStatus for job 'z': )get_job_infostatusr   	SUCCEEDEDr4   STOPPEDr   warningFAILEDr7   messageprint)r   r8   infor    r    r!   _log_job_statusO   s    



rE   c                    s2   |  |2 z3 d H W }t|dd q6 t| |S )N )end)tail_job_logsrC   rE   )r   r8   linesr    r    r!   
_tail_logsa   s
   
rJ   jobc                   C   s   dS )z'Submit, stop, delete, or list Ray jobs.Nr    r    r    r    r!   job_cli_grouph   s   rL   z	--addresszAddress of the Ray cluster to connect to. Can also be specified using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS).)typedefaultrequiredhelpz--job-idz*DEPRECATED: Use `--submission-id` instead.z--submission-idzMSubmission ID to specify for the job. If not provided, one will be generated.z--runtime-envz>Path to a local YAML file containing a runtime_env definition.z--runtime-env-jsonz'JSON-serialized runtime_env dictionary.z--working-dirzDirectory containing files that your job will run in. Can be a local directory or a remote URI to a .zip file (S3, GS, HTTP). If specified, this overrides the option in `--runtime-env`.z--metadata-jsonz<JSON-serialized dictionary of metadata to attach to the job.z--entrypoint-num-cpusz|the quantity of CPU cores to reserve for the entrypoint command, separately from any tasks or actors that are launched by it)rO   rM   rP   z--entrypoint-num-gpuszwthe quantity of GPUs to reserve for the entrypoint command, separately from any tasks or actors that are launched by itz--entrypoint-memoryzwthe amount of memory to reserve for the entrypoint command, separately from any tasks or actors that are launched by itz--entrypoint-resourcesza JSON-serialized dictionary mapping resource name to resource quantity describing resources to reserve for the entrypoint command, separately from any tasks or actors that are launched by itz--entrypoint-label-selectorza JSON-serialized dictionary mapping label keys to selector strings describing placement constraints for the entrypoint commandz	--no-waitz:If set, will not stream logs and wait for the job to exit.)is_flagrM   rN   rP   
entrypoint)nargsrO   rM   submission_idruntime_envruntime_env_jsonmetadata_jsonworking_direntrypoint_num_cpusentrypoint_num_gpusentrypoint_memoryentrypoint_resourcesentrypoint_label_selectorno_waitc                 C   sj  |rt d |durt|t tdd}|durt|t tdd}|dur+t|t tdd}|p.|}tjtjv rMt	tjtj | |||||||||	|
|||d t
| d||d	}t|||d
}|jt||||||	|
||d	}td| d t dt t d t   t td|  W d   n1 sw   Y  t d t   t td|  W d   n1 sw   Y  t d t   t td|  W d   n1 sw   Y  W d   n1 sw   Y  t   t   | }|s3t|dkr(t d t t||}|tjkr&td dS dS t d| d dS dS )al  Submits a job to be run on the cluster.

    By default (if --no-wait is not set), streams logs to stdout until the job finishes.
    If the job succeeded, exits with 0. If it failed, exits with 1.

    Example:
        `ray job submit -- python my_script.py --arg=val`

    Args:
        address: Job submission server address.
        job_id: DEPRECATED. Use submission_id instead.
        submission_id: Submission ID for the job.
        runtime_env: Path to a runtime_env YAML file.
        runtime_env_json: JSON-serialized runtime_env dictionary.
        metadata_json: JSON-serialized metadata dictionary.
        working_dir: Working directory for the job.
        entrypoint: Entrypoint command.
        entrypoint_num_cpus: CPU cores to reserve.
        entrypoint_num_gpus: GPUs to reserve.
        entrypoint_memory: Memory to reserve.
        entrypoint_resources: JSON-serialized custom resources dict.
        entrypoint_label_selector: JSON-serialized label selector dict.
        no_wait: Do not wait for job completion.
        verify: TLS verification flag or path.
        headers: JSON-serialized headers.
    zB--job-id option is deprecated. Please use --submission-id instead.Nzentrypoint-resources)command_argzentrypoint-label-selectorzmetadata-json)r   r8   rU   rV   rW   rX   rY   rR   rZ   r[   r\   r]   r^   r_   T)r   r   r   )rV   rW   rY   )	rR   rU   rV   metadatarZ   r[   r\   r]   r^   r9   z' submitted successfullyz
Next stepszQuery the logs of the job:zray job logs zQuery the status of the job:zray job status zRequest the job to be stopped:zray job stop r   z:Tailing logs until the job exits (disable with --no-wait):   z7Tailing logs is not enabled for job sdk client version z<. Please upgrade Ray to the latest version for this feature.)r   r@   r   r   r
   ray_constantsRAY_JOB_SUBMIT_HOOKr$   r%   r	   r"   r   
submit_jobr   r4   grouprC   indentedboldr/   flushget_versionintr   run_until_completerJ   r   rA   sysexit)r   r8   rU   rV   rW   rX   rY   rR   rZ   r[   r\   r]   r^   r_   r   r   r   final_runtime_envsdk_version
job_statusr    r    r!   submitn   s    





rr   zjob-id)rM   stable)	stabilityc                 C   s   t | ||d}t|| dS )z`Queries for the current status of a job.

    Example:
        `ray job status <my_job_id>`
    r   N)r"   rE   r   r8   r   r   r   r    r    r!   r=   k  s   r=   z*If set, will not wait for the job to exit.c                 C   s   t | ||d}td| d || |rdS td| d 	 ||}|tjtjtjhv r9t	|| dS td|  t
d	 q#)
zMAttempts to stop a job.

    Example:
        `ray job stop <my_job_id>`
    r   zAttempting to stop job ''NzWaiting for job 'z#' to exit (disable with --no-wait):Tz Job has not exited yet. Status: rb   )r"   r   rC   stop_jobget_job_statusr   r?   r>   rA   rE   timesleep)r   r_   r8   r   r   r   r=   r    r    r!   stop  s    "




r{   c                 C   s.   t | ||d}|| td| d dS )a  Deletes a stopped job and its associated data from memory.

    Only supported for jobs that are already in a terminal state.
    Fails with exit code 1 if the job is not already stopped.
    Does not delete job logs from disk.
    Submitting a job with the same submission ID as a previously
    deleted job is not supported and may lead to unexpected behavior.

    Example:
        ray job delete <my_job_id>
    r   r9   z' deleted successfullyN)r"   
delete_jobr   rC   ru   r    r    r!   delete  s    
r}   z-fz--followz)If set, follow the logs (like `tail -f`).followc                 C   sl   t | ||d}| }|r)t|dkrt t|| dS td| d dS tj|	|ddd dS )	zMGets the logs of a job.

    Example:
        `ray job logs <my_job_id>`
    r   r   z<Tailing logs is not enabled for the Jobs SDK client version z8. Please upgrade Ray to latest version for this feature.rF   T)rG   r;   N)
r"   rj   rk   r   rl   rJ   r   r@   rC   get_job_logs)r   r8   r~   r   r   r   rp   r    r    r!   logs  s   #r   c                 C   s*   t | ||d}tjt| dd dS )zWLists all running jobs and their information.

    Example:
        `ray job list`
    r   Tr:   N)r"   r   rC   pprintpformat	list_jobs)r   r   r   r   r    r    r!   list  s   r   )FNT)<r&   r$   r   rm   ry   
subprocessr   typingr   r   r   r   r   clickray._private.ray_constants_privaterc   ray._common.utilsr   r	   ray._private.utilsr
   r   "ray.autoscaler._private.cli_loggerr   r   r   #ray.dashboard.modules.dashboard_sdkr   #ray.dashboard.modules.job.cli_utilsr   ray.dashboard.modules.job.utilsr   ray.job_submissionr   r   ray.util.annotationsr   strboolr"   r   r4   r7   rE   rJ   rf   rL   commandoptionfloatrk   argumentUNPROCESSEDrr   r=   r{   r}   r   r   r    r    r    r!   <module>   s,   

"


	

. 



!




4