o
    `۷i[                     @   s  d dl Z d dlZd dlZd dlZd dlmZmZmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZ d dlmZ d dlmZmZmZ d dlmZ d d	lmZmZ d d
lmZ d dlm Z  d dl!m"Z" dZ#dZ$e dd Z%dZ&dZ'e(e)Z*e"ddG dd de+e	Z,e"ddG dd de+e	Z-e"ddeG dd dZ.G dd dZ/de+dee+e+f fdd Z0d!e+d"e+de+fd#d$Z1d%ee+ef d&edefd'd(Z2eG d)d* d*Z3eG d+d, d,Z4eG d-d. d.Z5eG d/d0 d0Z6eG d1d2 d2Z7dS )3    N)asdict	dataclassreplace)Enum)Path)AnyDictOptionalTupleUnion)ray_constants)EventLogTypecheck_export_api_enabledget_export_event_logger)	parse_uri)RAY_INTERNAL_NAMESPACE_PREFIX	GcsClient)ExportEvent)ExportSubmissionJobEventData)	PublicAPIjob_submission_idjob_name
job_actor_{job_id}SUPERVISOR_ACTOR_RAY_NAMESPACEzjob-driver-{submission_id}.logstable)	stabilityc                   @   s@   e Zd ZdZdZdZdZdZdZde	fdd	Z
defd
dZdS )	JobStatusz2An enumeration for describing the status of a job.PENDINGRUNNINGSTOPPED	SUCCEEDEDFAILEDreturnc                 C   s   | j  S Nvalueself r)   V/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/dashboard/modules/job/common.py__str__4   s   zJobStatus.__str__c                 C   s
   | j dv S )a  Return whether or not this status is terminal.

        A terminal status is one that cannot transition to any other status.
        The terminal statuses are "STOPPED", "SUCCEEDED", and "FAILED".

        Returns:
            True if this status is terminal, otherwise False.
        >   r"   r    r!   r%   r'   r)   r)   r*   is_terminal7   s   
	zJobStatus.is_terminalN)__name__
__module____qualname____doc__r   r   r    r!   r"   strr+   boolr,   r)   r)   r)   r*   r   %   s    r   c                   @   s0   e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
S )JobErrorTypez6An enumeration for describing the error type of a job.RUNTIME_ENV_SETUP_FAILURE"JOB_SUPERVISOR_ACTOR_START_TIMEOUT"JOB_SUPERVISOR_ACTOR_START_FAILURE"JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE$JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILUREJOB_SUPERVISOR_ACTOR_DIED"JOB_ENTRYPOINT_COMMAND_START_ERRORJOB_ENTRYPOINT_COMMAND_ERRORN)r-   r.   r/   r0   r4   r5   r6   r7   r8   r9   r:   r;   r)   r)   r)   r*   r3   C   s    r3   c                   @   sV  e Zd ZU dZeed< eed< dZee ed< dZ	ee
 ed< dZee ed< dZee ed< dZeeeef  ed	< dZeeeef  ed
< dZeeeef  ed< dZeeeef  ed< dZee ed< dZeeeef  ed< dZee ed< dZee ed< dZee ed< dd Zdeeef fddZedeeef ddfddZdS )JobInfozA class for recording information associated with a job and its execution.

    Please keep this in sync with the JobsAPIInfo proto in src/ray/protobuf/gcs.proto.
    status
entrypointNmessage
error_type
start_timeend_timemetadataruntime_enventrypoint_num_cpusentrypoint_num_gpusentrypoint_memoryentrypoint_resourcesdriver_agent_http_addressdriver_node_iddriver_exit_codec                 C   s  t | jtrt| j| _| jd u r| jtjkrXd| _t| jd uo$| jdk| jd uo-| jdk| j	d uo6| j	dk| j
d i fvgrF|  jd7  _| jd i fvrV|  jd7  _d S d S | jtjkrcd| _d S | jtjkrnd| _d S | jtjkryd| _d S | jtjkrd| _d S d S d S )	NzJob has not started yet.r   z\ It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available.z< It may be waiting for the runtime environment to be set up.zJob is currently running.zJob was intentionally stopped.zJob finished successfully.zJob failed.)
isinstancer=   r1   r   r?   r   anyrE   rF   rG   rH   rD   r   r    r!   r"   r'   r)   r)   r*   __post_init__   sB   








zJobInfo.__post_init__r#   c                 C   sn   t | }t|d |d< |dr|d jnd|d< d|v r0|d dur-t|d |d< |d= t| |S )a%  Convert this object to a JSON-serializable dictionary.

        Note that the runtime_env field is converted to a JSON-serialized string
        and the field is renamed to runtime_env_json.

        Returns:
            A JSON-serializable dictionary representing the JobInfo object.
        r=   r@   NrD   runtime_env_json)r   r1   getr&   jsondumps)r(   	json_dictr)   r)   r*   to_json   s   

zJobInfo.to_jsonrS   c                 C   sh   t |d |d< |drt|d nd|d< d|v r-|d dur*t|d |d< |d= | di |S )a  Initialize this object from a JSON dictionary.

        Note that the runtime_env_json field is converted to a dictionary and
        the field is renamed to runtime_env.

        Args:
            json_dict: A JSON dictionary to use to initialize the JobInfo object.
        r=   r@   NrO   rD   r)   )r   rP   r3   rQ   loads)clsrS   r)   r)   r*   	from_json   s   zJobInfo.from_json)r-   r.   r/   r0   r   __annotations__r1   r?   r	   r@   r3   rA   intrB   rC   r   rD   r   rE   r   floatrF   rG   rH   rI   rJ   rK   rN   rT   classmethodrW   r)   r)   r)   r*   r<   Z   s*   
 " r<   c                   @   s2  e Zd ZdZe dZe dZ	d#dedee	 fddZ
			
d$de	dededee def
ddZde	deddfddZd%de	dedee fddZd%de	defddZ					
d&de	dedee	 dee dee deee	ef  dee fddZd%de	dedee fdd Zd%dedee	ef fd!d"ZdS )'JobInfoStorageClientzG
    Interface to put and get job data from the Internal KV store.
    	job_info_r   N
gcs_clientexport_event_log_dir_rootc                 C   s`   || _ d| _zttjjr|durttj|| _W dS W dS W dS  t	y/   t
d Y dS w )av  
        Initialize the JobInfoStorageClient which manages data in the internal KV store.
        Export Submission Job events are written when the KV store is updated if
        the feature flag is on and a export_event_log_dir_root is passed.
        export_event_log_dir_root doesn't need to be passed if the caller
        is not modifying data in the KV store.
        NzMUnable to initialize export event logger so no export events will be written.)_gcs_client#_export_submission_job_event_loggerr   r   
SourceTypeEXPORT_SUBMISSION_JOBr   r   SUBMISSION_JOB	Exceptionlogger	exception)r(   r^   r_   r)   r)   r*   __init__   s"   

zJobInfoStorageClient.__init__T   job_idjob_info	overwritetimeoutr#   c                    s   | j j| jj|d t|  |tj	|dI dH }|dks$|rAz| 
|| W |dkS  ty@   td Y |dkS w |dkS )aB  Put job info to the internal kv store.

        Args:
            job_id: The job id.
            job_info: The job info.
            overwrite: Whether to overwrite the existing job info.
            timeout: The timeout in seconds for the GCS operation.

        Returns:
            True if a new key is added.
        rj   	namespacerm   N   z0Error while writing job submission export event.)r`   async_internal_kv_putJOB_DATA_KEYformatencoderQ   rR   rT   r   KV_NAMESPACE_JOB"_write_submission_job_export_eventre   rf   rg   )r(   rj   rk   rl   rm   	added_numr)   r)   r*   put_info  s"   zJobInfoStorageClient.put_infoc                 C   s   | j sdS tjjj|jj}|du rt	|jj d dS |j
}t|||j|j|j|j|j|jt|j|j|j|jd}| j | dS )z
        Write Submission Job export event if _export_submission_job_event_logger
        exists. The logger will exist if the export API feature flag is enabled
        and a log directory was passed to JobInfoStorageClient.
        Nzb is not a valid ExportSubmissionJobEventData.JobStatus enum value. This event will not be written.)submission_job_idr=   r>   r?   rC   r@   rA   rB   rO   rI   rJ   rK   )ra   r   r   
DESCRIPTORvalues_by_namerP   r=   namerf   errornumberr>   r?   rC   r@   rA   rB   rQ   rR   rD   rI   rJ   rK   
send_event)r(   rj   rk   status_value_descriptor
job_statussubmission_event_datar)   r)   r*   rw   %  s6   

z7JobInfoStorageClient._write_submission_job_export_eventc                    sF   | j j| jj|d tj|dI d H }|d u rd S tt	
|S )Nrn   ro   )r`   async_internal_kv_getrs   rt   ru   r   rv   r<   rW   rQ   rU   )r(   rj   rm   serialized_infor)   r)   r*   get_infoM  s   zJobInfoStorageClient.get_infoc                    s0   | j j| jj|d dtj|dI d H  d S )Nrn   Fro   )r`   async_internal_kv_delrs   rt   ru   r   rv   )r(   rj   rm   r)   r)   r*   delete_infoX  s   z JobInfoStorageClient.delete_infor=   r?   rK   r@   jobinfo_replace_kwargsc           
         s   | j ||dI dH }|du rt }|j||||d |dur6||jkr-|j r-J dt|fi |}	n	td	ddi|}	| rLtt d |	_	| j
||	|dI dH  dS )
zAPuts or updates job status.  Sets end_time if status is terminal.)rm   N)r=   r?   rK   r@   Fz5Attempted to change job status from a terminal state.r>   zEntrypoint not found.i  r)   )r   dictupdater=   r,   r   r<   rY   timerB   ry   )
r(   rj   r=   r?   rK   r@   r   rm   old_infonew_infor)   r)   r*   
put_status`  s,   zJobInfoStorageClient.put_statusc                    s&   |  ||I d H }|d u rd S |jS r$   )r   r=   )r(   rj   rm   rk   r)   r)   r*   
get_status  s
   zJobInfoStorageClient.get_statusc                    s   j jj tjdI d H }dd |D }g }|D ]}|js(J d||tjd   qdt	ffdd t
tj fdd|D  I d H S )	Nro   c                 S   s   g | ]}|  qS r)   )decode.0rj   r)   r)   r*   
<listcomp>  s    z5JobInfoStorageClient.get_all_jobs.<locals>.<listcomp>z8Unexpected format for internal_kv key for Job submissionrj   c                    s     | I d H }| |fS r$   )r   )rj   rk   )r(   rm   r)   r*   get_job_info  s   z7JobInfoStorageClient.get_all_jobs.<locals>.get_job_infoc                    s   g | ]} |qS r)   r)   r   )r   r)   r*   r     s    )r`   async_internal_kv_keysJOB_DATA_KEY_PREFIXru   r   rv   
startswithappendlenr1   r   asynciogather)r(   rm   raw_job_ids_with_prefixesjob_ids_with_prefixesjob_idsjob_id_with_prefixr)   )r   r(   rm   r*   get_all_jobs  s&   "z!JobInfoStorageClient.get_all_jobsr$   )Tri   )ri   )NNNNri   )r-   r.   r/   r0   r   r   rs   r   r	   r1   rh   r<   r2   rY   ry   rw   r   r   r   r3   r   r   r   r   r   r)   r)   r)   r*   r\      sl    


!
!
(
$ r\   package_urir#   c                 C   s8   t | j}|dvrtd|  dt| \}}|j|fS )N>   .whl.zipzpackage_uri (z) does not end in .zip or .whl)r   suffix
ValueErrorr   r&   )r   r   protocolpackage_namer)   r)   r*   uri_to_http_components  s
   

r   r   r   c                 C   s   |  d| S )Nz://r)   )r   r   r)   r)   r*   http_uri_components_to_uri     r   	json_datarequest_typec                 C   s   |di | S )Nr)   r)   )r   r   r)   r)   r*   validate_request_type  r   r   c                   @   s   e Zd ZU eed< dZee ed< dZee ed< dZee	ee
f  ed< dZee	eef  ed< dZeeeef  ed< dZeeeef  ed< dZee ed	< dZee	eef  ed
< dZee	eef  ed< dd ZdS )JobSubmitRequestr>   Nsubmission_idrj   rD   rC   rE   rF   rG   rH   entrypoint_label_selectorc                 C   s  t | jtstdt| j | jd ur%t | jts%tdt| j | jd ur:t | jts:tdt| j | jd uret | jtsOtdt| j | j	 D ]}t |tsdtdt| qT| j
d urt | j
tsztdt| j
 | j
	 D ]}t |tstdt| q| j
 D ]}t |tstdt| q| jd urt | jttfstd	t| j | jd urt | jttfstd
t| j | jd urt | jtstdt| j | jd ur1t | jtstdt| j | j	 D ]}t |tstdt| q| j D ]}t |ttfs/tdt| q| jd urqt | jtsHtdt| j | j D ]%\}}t |ts`tdt| t |tsotdt| qMd S d S )Nz!entrypoint must be a string, got z0submission_id must be a string if provided, got z)job_id must be a string if provided, got z runtime_env must be a dict, got z&runtime_env keys must be strings, got zmetadata must be a dict, got z#metadata keys must be strings, got z%metadata values must be strings, got z*entrypoint_num_cpus must be a number, got z*entrypoint_num_gpus must be a number, got z*entrypoint_memory must be an integer, got z)entrypoint_resources must be a dict, got z/entrypoint_resources keys must be strings, got z1entrypoint_resources values must be numbers, got z.entrypoint_label_selector must be a dict, got z4entrypoint_label_selector keys must be strings, got z6entrypoint_label_selector values must be strings, got )rL   r>   r1   	TypeErrortyper   rj   rD   r   keysrC   valuesrE   rY   rZ   rF   rG   rH   r   items)r(   kvr)   r)   r*   rN     s   






zJobSubmitRequest.__post_init__)r-   r.   r/   r1   rX   r   r	   rj   rD   r   r   rC   rE   r   rY   rZ   rF   rG   rH   r   rN   r)   r)   r)   r*   r     s   
 r   c                   @   s   e Zd ZU eed< eed< dS )JobSubmitResponserj   r   Nr-   r.   r/   r1   rX   r)   r)   r)   r*   r   =  s   
 r   c                   @      e Zd ZU eed< dS )JobStopResponsestoppedNr-   r.   r/   r2   rX   r)   r)   r)   r*   r   D     
 r   c                   @   r   )JobDeleteResponsedeletedNr   r)   r)   r)   r*   r   I  r   r   c                   @   r   )JobLogsResponselogsNr   r)   r)   r)   r*   r   O  r   r   )8r   rQ   loggingr   dataclassesr   r   r   enumr   pathlibr   typingr   r   r	   r
   r   ray._privater   &ray._private.event.export_event_loggerr   r   r   "ray._private.runtime_env.packagingr   ray._rayletr   r   #ray.core.generated.export_event_pb2r   2ray.core.generated.export_submission_job_event_pb2r   ray.util.annotationsr   JOB_ID_METADATA_KEYJOB_NAME_METADATA_KEYJOB_ACTOR_NAME_TEMPLATEr   JOB_LOGS_PATH_TEMPLATE	getLoggerr-   rf   r1   r   r3   r<   r\   r   r   r   r   r   r   r   r   r)   r)   r)   r*   <module>   sZ    
  F
 	