o
    biNQ                     @   s  d dl Z d dlZd dlZd dlZd dlmZmZmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZ d dlmZ d dlmZmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dl m!Z! dZ"dZ#ej$ dd Z%dZ&dZ'e(e)Z*e!ddG dd de+e	Z,e!ddeG dd dZ-G dd dZ.de+dee+e+f fddZ/de+d e+de+fd!d"Z0d#ee+ef d$edefd%d&Z1eG d'd( d(Z2eG d)d* d*Z3eG d+d, d,Z4eG d-d. d.Z5eG d/d0 d0Z6dS )1    N)asdict	dataclassreplace)Enum)Path)AnyDictOptionalTupleUnion)ray_constants)EventLogTypecheck_export_api_enabledget_export_event_logger)	parse_uri)	GcsClient)ExportEvent)ExportSubmissionJobEventData)	PublicAPIjob_submission_idjob_name
job_actor_{job_id}SUPERVISOR_ACTOR_RAY_NAMESPACEzjob-driver-{submission_id}.logstable)	stabilityc                   @   s@   e Zd ZdZdZdZdZdZdZde	fdd	Z
defd
dZdS )	JobStatusz2An enumeration for describing the status of a job.PENDINGRUNNINGSTOPPED	SUCCEEDEDFAILEDreturnc                 C   s   | j  S Nvalueself r(   T/home/ubuntu/.local/lib/python3.10/site-packages/ray/dashboard/modules/job/common.py__str__6   s   zJobStatus.__str__c                 C   s
   | j dv S )a  Return whether or not this status is terminal.

        A terminal status is one that cannot transition to any other status.
        The terminal statuses are "STOPPED", "SUCCEEDED", and "FAILED".

        Returns:
            True if this status is terminal, otherwise False.
        >   r!   r   r    r$   r&   r(   r(   r)   is_terminal9   s   
	zJobStatus.is_terminalN)__name__
__module____qualname____doc__r   r   r   r    r!   strr*   boolr+   r(   r(   r(   r)   r   '   s    r   c                   @   sV  e Zd ZU dZeed< eed< dZee ed< dZ	ee ed< dZ
ee ed< dZee ed< dZeeeef  ed	< dZeeeef  ed
< dZeeeef  ed< dZeeeef  ed< dZee ed< dZeeeef  ed< dZee ed< dZee ed< dZee ed< dd Zdeeef fddZedeeef ddfddZdS )JobInfozA class for recording information associated with a job and its execution.

    Please keep this in sync with the JobsAPIInfo proto in src/ray/protobuf/gcs.proto.
    status
entrypointNmessage
error_type
start_timeend_timemetadataruntime_enventrypoint_num_cpusentrypoint_num_gpusentrypoint_memoryentrypoint_resourcesdriver_agent_http_addressdriver_node_iddriver_exit_codec                 C   s  t | jtrt| j| _| jd u r| jtjkrXd| _t| jd uo$| jdk| jd uo-| jdk| j	d uo6| j	dk| j
d i fvgrF|  jd7  _| jd i fvrV|  jd7  _d S d S | jtjkrcd| _d S | jtjkrnd| _d S | jtjkryd| _d S | jtjkrd| _d S d S d S )	NzJob has not started yet.r   z\ It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available.z< It may be waiting for the runtime environment to be set up.zJob is currently running.zJob was intentionally stopped.zJob finished successfully.zJob failed.)
isinstancer3   r0   r   r5   r   anyr;   r<   r=   r>   r:   r   r   r    r!   r&   r(   r(   r)   __post_init__p   sB   








zJobInfo.__post_init__r"   c                 C   sR   t | }t|d |d< d|v r"|d durt|d |d< |d= t| |S )a%  Convert this object to a JSON-serializable dictionary.

        Note that the runtime_env field is converted to a JSON-serialized string
        and the field is renamed to runtime_env_json.

        Returns:
            A JSON-serializable dictionary representing the JobInfo object.
        r3   r:   Nruntime_env_json)r   r0   jsondumps)r'   	json_dictr(   r(   r)   to_json   s   

zJobInfo.to_jsonrH   c                 C   sJ   t |d |d< d|v r|d durt|d |d< |d= | di |S )a  Initialize this object from a JSON dictionary.

        Note that the runtime_env_json field is converted to a dictionary and
        the field is renamed to runtime_env.

        Args:
            json_dict: A JSON dictionary to use to initialize the JobInfo object.
        r3   rE   Nr:   r(   )r   rF   loads)clsrH   r(   r(   r)   	from_json   s   zJobInfo.from_json)r,   r-   r.   r/   r   __annotations__r0   r5   r	   r6   r7   intr8   r9   r   r:   r   r;   r   floatr<   r=   r>   r?   r@   rA   rD   rI   classmethodrL   r(   r(   r(   r)   r2   F   s*   
 " r2   c                   @   s  e Zd ZdZej dZe dZ	d"dede	e
 fddZ		d#d
e
dededefddZd
e
deddfddZd$d
e
dede	e fddZd$d
e
defddZ			d%d
e
dede	e
 de	e de	ee
ef  f
ddZd
e
de	e fddZd$dedee
ef fd d!ZdS )&JobInfoStorageClientzG
    Interface to put and get job data from the Internal KV store.
    	job_info_r   N
gcs_clientexport_event_log_dir_rootc                 C   s`   || _ d| _zttjjr|durttj|| _W dS W dS W dS  t	y/   t
d Y dS w )av  
        Initialize the JobInfoStorageClient which manages data in the internal KV store.
        Export Submission Job events are written when the KV store is updated if
        the feature flag is on and a export_event_log_dir_root is passed.
        export_event_log_dir_root doesn't need to be passed if the caller
        is not modifying data in the KV store.
        NzMUnable to initialize export event logger so no export events will be written.)_gcs_client#_export_submission_job_event_loggerr   r   
SourceTypeEXPORT_SUBMISSION_JOBr   r   SUBMISSION_JOB	Exceptionlogger	exception)r'   rS   rT   r(   r(   r)   __init__   s"   

zJobInfoStorageClient.__init__Tjob_idjob_info	overwriter"   c                    s   | j j| jj|d t|  |tj	dI dH }|dks#|r@z| 
|| W |dkS  ty?   td Y |dkS w |dkS )zPut job info to the internal kv store.

        Args:
            job_id: The job id.
            job_info: The job info.
            overwrite: Whether to overwrite the existing job info.

        Returns:
            True if a new key is added.
        r^   )	namespaceN   z0Error while writing job submission export event.)rU   async_internal_kv_putJOB_DATA_KEYformatencoderF   rG   rI   r   KV_NAMESPACE_JOB"_write_submission_job_export_eventrZ   r[   r\   )r'   r^   r_   r`   	added_numr(   r(   r)   put_info   s    zJobInfoStorageClient.put_infoc                 C   s   | j sdS tjjj|jj}|du rt	|jj d dS |j
}t|||j|j|j|j|j|jt|j|j|j|jd}| j | dS )z
        Write Submission Job export event if _export_submission_job_event_logger
        exists. The logger will exist if the export API feature flag is enabled
        and a log directory was passed to JobInfoStorageClient.
        Nzb is not a valid ExportSubmissionJobEventData.JobStatus enum value. This event will not be written.)submission_job_idr3   r4   r5   r9   r6   r7   r8   rE   r?   r@   rA   )rV   r   r   
DESCRIPTORvalues_by_namegetr3   namer[   errornumberr4   r5   r9   r6   r7   r8   rF   rG   r:   r?   r@   rA   
send_event)r'   r^   r_   status_value_descriptor
job_statussubmission_event_datar(   r(   r)   ri     s6   

z7JobInfoStorageClient._write_submission_job_export_event   timeoutc                    sF   | j j| jj|d tj|dI d H }|d u rd S tt	
|S )Nra   rb   rx   )rU   async_internal_kv_getre   rf   rg   r   rh   r2   rL   rF   rJ   )r'   r^   rx   serialized_infor(   r(   r)   get_info,  s   zJobInfoStorageClient.get_infoc                    s0   | j j| jj|d dtj|dI d H  d S )Nra   Fry   )rU   async_internal_kv_delre   rf   rg   r   rh   )r'   r^   rx   r(   r(   r)   delete_info7  s   z JobInfoStorageClient.delete_infor3   r5   rA   jobinfo_replace_kwargsc                    s   |  |I dH }|du rt }|j|||d |dur3||jkr*|j r*J dt|fi |}n	tdddi|}| rItt d |_	| 
||I dH  dS )	zAPuts or updates job status.  Sets end_time if status is terminal.N)r3   r5   rA   Fz5Attempted to change job status from a terminal state.r4   zEntrypoint not found.i  r(   )r|   dictupdater3   r+   r   r2   rN   timer8   rk   )r'   r^   r3   r5   rA   r   old_infonew_infor(   r(   r)   
put_status?  s&   
zJobInfoStorageClient.put_statusc                    s$   |  |I d H }|d u rd S |jS r#   )r|   r3   )r'   r^   r_   r(   r(   r)   
get_status^  s
   zJobInfoStorageClient.get_statusc                    s   j jj tjdI d H }dd |D }g }|D ]}|js(J d||tjd   qdt	ffdd t
tj fdd|D  I d H S )	Nry   c                 S   s   g | ]}|  qS r(   )decode.0r^   r(   r(   r)   
<listcomp>k  s    z5JobInfoStorageClient.get_all_jobs.<locals>.<listcomp>z8Unexpected format for internal_kv key for Job submissionr^   c                    s     | I d H }| |fS r#   )r|   )r^   r_   )r'   rx   r(   r)   get_job_infou  s   z7JobInfoStorageClient.get_all_jobs.<locals>.get_job_infoc                    s   g | ]} |qS r(   r(   r   )r   r(   r)   r   y  s    )rU   async_internal_kv_keysJOB_DATA_KEY_PREFIXrg   r   rh   
startswithappendlenr0   r   asynciogather)r'   rx   raw_job_ids_with_prefixesjob_ids_with_prefixesjob_idsjob_id_with_prefixr(   )r   r'   rx   r)   get_all_jobse  s&   "z!JobInfoStorageClient.get_all_jobsr#   )T)rw   )NNN)r,   r-   r.   r/   r   RAY_INTERNAL_NAMESPACE_PREFIXr   re   r   r	   r0   r]   r2   r1   rk   ri   rN   r|   r~   r   r   r   r   r   r   r(   r(   r(   r)   rQ      sZ    



(
 rQ   package_urir"   c                 C   s8   t | j}|dvrtd|  dt| \}}|j|fS )N>   .whl.zipzpackage_uri (z) does not end in .zip or .whl)r   suffix
ValueErrorr   r%   )r   r   protocolpackage_namer(   r(   r)   uri_to_http_components|  s
   

r   r   r   c                 C   s   |  d| S )Nz://r(   )r   r   r(   r(   r)   http_uri_components_to_uri     r   	json_datarequest_typec                 C   s   |di | S )Nr(   r(   )r   r   r(   r(   r)   validate_request_type  r   r   c                   @   s   e Zd ZU eed< dZee ed< dZee ed< dZee	ee
f  ed< dZee	eef  ed< dZeeeef  ed< dZeeeef  ed< dZee ed	< dZee	eef  ed
< dd ZdS )JobSubmitRequestr4   Nsubmission_idr^   r:   r9   r;   r<   r=   r>   c                 C   sj  t | jtstdt| j | jd ur%t | jts%tdt| j | jd ur:t | jts:tdt| j | jd uret | jtsOtdt| j | j	 D ]}t |tsdtdt| qT| j
d urt | j
tsztdt| j
 | j
	 D ]}t |tstdt| q| j
 D ]}t |tstdt| q| jd urt | jttfstd	t| j | jd urt | jttfstd
t| j | jd urt | jtstdt| j | jd ur1t | jtstdt| j | j	 D ]}t |tstdt| q| j D ]}t |ttfs/tdt| qd S d S )Nz!entrypoint must be a string, got z0submission_id must be a string if provided, got z)job_id must be a string if provided, got z runtime_env must be a dict, got z&runtime_env keys must be strings, got zmetadata must be a dict, got z#metadata keys must be strings, got z%metadata values must be strings, got z*entrypoint_num_cpus must be a number, got z*entrypoint_num_gpus must be a number, got z*entrypoint_memory must be an integer, got z)entrypoint_resources must be a dict, got z/entrypoint_resources keys must be strings, got z1entrypoint_resources values must be numbers, got )rB   r4   r0   	TypeErrortyper   r^   r:   r   keysr9   valuesr;   rN   rO   r<   r=   r>   )r'   kvr(   r(   r)   rD     s   






zJobSubmitRequest.__post_init__)r,   r-   r.   r0   rM   r   r	   r^   r:   r   r   r9   r;   r   rN   rO   r<   r=   r>   rD   r(   r(   r(   r)   r     s   
 r   c                   @   s   e Zd ZU eed< eed< dS )JobSubmitResponser^   r   Nr,   r-   r.   r0   rM   r(   r(   r(   r)   r     s   
 r   c                   @      e Zd ZU eed< dS )JobStopResponsestoppedNr,   r-   r.   r1   rM   r(   r(   r(   r)   r   	     
 r   c                   @   r   )JobDeleteResponsedeletedNr   r(   r(   r(   r)   r     r   r   c                   @   r   )JobLogsResponselogsNr   r(   r(   r(   r)   r     r   r   )7r   rF   loggingr   dataclassesr   r   r   enumr   pathlibr   typingr   r   r	   r
   r   ray._privater   &ray._private.event.export_event_loggerr   r   r   "ray._private.runtime_env.packagingr   ray._rayletr   #ray.core.generated.export_event_pb2r   2ray.core.generated.export_submission_job_event_pb2r   ray.util.annotationsr   JOB_ID_METADATA_KEYJOB_NAME_METADATA_KEYr   JOB_ACTOR_NAME_TEMPLATEr   JOB_LOGS_PATH_TEMPLATE	getLoggerr,   r[   r0   r   r2   rQ   r   r   r   r   r   r   r   r   r(   r(   r(   r)   <module>   sT    
z ;
s