o
    biZ                  
   @   sV  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl m	Z	m
Z
 d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d dlmZ erbd d	lmZ d d
lmZ d dl Z d dl!Z d dl"m#  m$Z$ d dl%m#  m&Z& d dl'm(  m)Z) d dlm*Z* d dl+m,Z, d dl-m.Z.m/Z/ d dl0m1Z1 zej2Z2W n e3y   ej4Z2Y nw e5e6Z7G dd deZ8G dd de9Z:G dd de j;Z<eG dd dZ=G dd de j;Z>G dd de j;Z?dd Z@dd ZAdd  ZBd!d" ZCd#d$d%eeeDeeeDef  f  fd&d'ZEG d(d) d)ejFZGd*d+ ZHd,d- ZIdNd.d/ZJG d0d1 d1eKZL	 eKeMeNeDeOePeQeRdeSh	ZTd2d3 ZUdOd5d6ZVG d7d8 d8e	d9ZWG d:d; d;eWeZXG d<d= d=eWeZYeWZ D ]Z[eT\e[ qidPd?d@Z]dAeDfdBdCZ^dAeeD d%eDfdDdEZ_dAeeD d%eDfdFdGZ`dHeeD dIeeD d%eeD fdJdKZadLdM ZbdS )Q    N)ABCMetaabstractmethod)	b64decode)MappingSequence)	dataclass)IntEnum)TYPE_CHECKINGAnyDictListOptional)binary_to_hex)GetNodeStatsReply)Version)get_or_create_event_loop)
GcsChannel)&check_dashboard_dependencies_installedsplit_address)	GcsClientc                   @   s   e Zd ZdZdZdZdS )HTTPStatusCode   i  i  N)__name__
__module____qualname__OK	NOT_FOUNDINTERNAL_ERROR r   r   G/home/ubuntu/.local/lib/python3.10/site-packages/ray/dashboard/utils.pyr   -   s    r   c                   @   s   e Zd ZdS )FrontendNotFoundErrorN)r   r   r   r   r   r   r   r    8   s    r    c                   @   s@   e Zd Zdd Zejdd Zeejdd Z	e
dd Zd	S )
DashboardAgentModulec                 C   s   || _ |j| _dS )z
        Initialize current module when DashboardAgent loading modules.
        :param dashboard_agent: The DashboardAgent instance.
        N)_dashboard_agentsession_name)selfdashboard_agentr   r   r   __init__=   s   zDashboardAgentModule.__init__c                       dS )z
        Run the module in an asyncio loop. An agent module can provide
        servicers to the server.
        :param server: Asyncio GRPC server, or None if ray is minimal.
        Nr   )r$   serverr   r   r   runE       zDashboardAgentModule.runc                   C      dS z
        Return True if the module is minimal, meaning it
        should work with `pip install ray` that doesn't requires additional
        dependencies.
        Nr   r   r   r   r   is_minimal_moduleM       z&DashboardAgentModule.is_minimal_modulec                 C      | j jS N)r"   gcs_addressr$   r   r   r   r1   V      z DashboardAgentModule.gcs_addressN)r   r   r   r&   abcr   r)   staticmethodabstractclassmethodr-   propertyr1   r   r   r   r   r!   <   s    
r!   c                   @   s^   e Zd ZU eed< eed< eed< eed< eed< eed< eed< eed< eed	< eed
< dS )DashboardHeadModuleConfigminimalcluster_id_hexr#   r1   log_dirtemp_dirsession_dirip	http_host	http_portN)r   r   r   bool__annotations__strintr   r   r   r   r8   [   s   
 r8   c                   @   s   e Zd ZdefddZedd Zedd Zedd	 Zed
d Z	edd Z
edd Zedd Zedd Zedd Zedd Zedd Zedd Zejdd Zeejdd Zd S )!DashboardHeadModuleconfigc                 C   s   || _ d| _d| _d| _dS )z
        Initialize current module when DashboardHead loading modules.
        :param config: The DashboardHeadModuleConfig instance.
        N)_config_gcs_client_aiogrpc_gcs_channel_http_session)r$   rF   r   r   r   r&   j   s   
zDashboardHeadModule.__init__c                 C   r/   r0   )rG   r9   r2   r   r   r   r9   t   r3   zDashboardHeadModule.minimalc                 C   r/   r0   )rG   r#   r2   r   r   r   r#   x   r3   z DashboardHeadModule.session_namec                 C   r/   r0   )rG   r1   r2   r   r   r   r1   |   r3   zDashboardHeadModule.gcs_addressc                 C   r/   r0   )rG   r;   r2   r   r   r   r;      r3   zDashboardHeadModule.log_dirc                 C   r/   r0   )rG   r<   r2   r   r   r   r<      r3   zDashboardHeadModule.temp_dirc                 C   r/   r0   )rG   r=   r2   r   r   r   r=      r3   zDashboardHeadModule.session_dirc                 C   r/   r0   )rG   r>   r2   r   r   r   r>      r3   zDashboardHeadModule.ipc                 C   r/   r0   )rG   r?   r2   r   r   r   r?      r3   zDashboardHeadModule.http_hostc                 C   r/   r0   )rG   r@   r2   r   r   r   r@      r3   zDashboardHeadModule.http_portc                 C   s`   | j jrJ ddd l}| jd ur| jS t|jtdk r(|jt d| _| jS | | _| jS )Nz%http_session accessed in minimal Ray.r   z4.0.0)loop)rG   r9   aiohttprJ   r   __version__ClientSessionr   )r$   rL   r   r   r   http_session   s   

z DashboardHeadModule.http_sessionc                 C   s:   | j d u rt| jj| jjd| _ t st| j  | j S )N)address
cluster_id)rH   r   rG   r1   r:   internal_kv_internal_kv_initialized_initialize_internal_kvr2   r   r   r   
gcs_client   s   
zDashboardHeadModule.gcs_clientc                 C   s>   | j jrd S | jd u rt| j jdd}|  | | _| jS )NT)r1   aio)rG   r9   rI   r   r1   connectchannel)r$   gcs_channelr   r   r   aiogrpc_gcs_channel   s   

z'DashboardHeadModule.aiogrpc_gcs_channelc                    r'   )zo
        Run the module in an asyncio loop. A head module can provide
        servicers to the server.
        Nr   r2   r   r   r   r)      r*   zDashboardHeadModule.runc                   C   r+   r,   r   r   r   r   r   r-      r.   z%DashboardHeadModule.is_minimal_moduleN)r   r   r   r8   r&   r7   r9   r#   r1   r;   r<   r=   r>   r?   r@   rO   rU   rZ   r4   r   r)   r5   r6   r-   r   r   r   r   rE   i   s>    















rE   c                   @   sB   e Zd ZdZddedeej fddZe	dd Z
ed	d
 ZdS )RateLimitedModulea  Simple rate limiter

    Inheriting from this class and decorate any class methods will
    apply simple rate limit.
    It will limit the maximal number of concurrent invocations of **all** the
    methods decorated.

    The below Example class will only allow 10 concurrent calls to A() and B()

    E.g.:

        class Example(RateLimitedModule):
            def __init__(self):
                super().__init__(max_num_call=10)

            @RateLimitedModule.enforce_max_concurrent_calls
            async def A():
                ...

            @RateLimitedModule.enforce_max_concurrent_calls
            async def B():
                ...

            async def limit_handler_(self):
                raise RuntimeError("rate limited reached!")

    Nmax_num_callloggerc                 C   s   || _ d| _|| _dS )z
        Args:
            max_num_call: Maximal number of concurrent invocations of all decorated
                functions in the instance.
                Setting to -1 will disable rate limiting.

            logger: Logger
        r   N)max_num_call_	num_call_logger_)r$   r\   r]   r   r   r   r&      s   	
zRateLimitedModule.__init__c                    s   t   fdd}|S )a  Decorator to enforce max number of invocations of the decorated func

        NOTE: This should be used as the innermost decorator if there are multiple
        ones.

        E.g., when decorating functions already with @routes.get(...), this must be
        added below then the routes decorators:
            ```
            @routes.get('/')
            @RateLimitedModule.enforce_max_concurrent_calls
            async def fn(self):
                ...

            ```
        c              	      s   | j dkr | j| j kr | jr| jd| j   |  I d H S |  jd7  _z | g|R i |I d H }W |  jd8  _|S |  jd8  _w )Nr   z Max concurrent requests reached=   )r^   r_   r`   warninglimit_handler_)r$   argskwargsretfuncr   r   async_wrapper
  s   
zERateLimitedModule.enforce_max_concurrent_calls.<locals>.async_wrapper	functoolswraps)rh   ri   r   rg   r   enforce_max_concurrent_calls   s   z.RateLimitedModule.enforce_max_concurrent_callsc                    r'   )zCHandler that is invoked when max number of concurrent calls reachedNr   r2   r   r   r   rc     r*   z RateLimitedModule.limit_handler_r0   )r   r   r   __doc__rD   r   loggingLoggerr&   r5   rm   r   rc   r   r   r   r   r[      s    
$r[   c                    s    fdd}|S )z!A decorator for dashboard module.c                    s
    | _ | S r0   )__ray_dashboard_module_enable__)clsenabler   r   _cls_wrapper%  s   z&dashboard_module.<locals>._cls_wrapperr   )rt   ru   r   rs   r   dashboard_module"  s   rv   c           	      C   s   t d| j  ddl}t  }t|jjj	|jjjd D ]2\}}}zt
| W q tyQ } zt d| d|  |sGt d |W Y d}~qd}~ww g }|  D ]}t|dd	saqX|rh| shqX|| qXt d
|  |S )zN
    Get all importable modules that are subclass of a given module type.
    zGet all modules by type: r   N.zModule z cannot be loaded because we cannot import all dependencies. Install this module using `pip install 'ray[default]'` for the full dashboard functionality. Error: zQAlthough `pip install 'ray[default]'` is downloaded, module couldn't be imported`rq   TzAvailable modules: )r]   infor   ray.dashboard.modulesr   pkgutilwalk_packages	dashboardmodules__path__	importlibimport_moduleModuleNotFoundError__subclasses__getattrr-   append)	module_typeray should_only_load_minimal_modulesmodule_loadernameispkgeimported_modulesmr   r   r   get_all_modules,  s>   r   c                 C   s   | t  ddd  S )Ni  ra   )datetimetotal_seconds)dtr   r   r   to_posix_timeU  s   r   c                 C   s(   t | tr| S | d\}}|t|fS )N:)
isinstancetuplesplitrD   )rP   r>   portr   r   r   address_tupleY  s   
r   messager   returnc                    s2   h d | j }t|  } fdd|D |d< |S )N>	   jobIdtaskIdactorIdcallerIdrayletIdworkerIdparentTaskIdsourceActorIdplacementGroupIdc                    s   g | ]	}t | d dqS )T)$always_print_fields_with_no_presence)message_to_dict).0r   decode_keysr   r   
<listcomp>p  s    z&node_stats_to_dict.<locals>.<listcomp>coreWorkersStats)core_workers_statsr   )r   r   resultr   r   r   node_stats_to_dict`  s   


r   c                   @   s   e Zd Zdd ZdS )CustomEncoderc                 C   s2   t |tr	t|S t |tr| S tj| |S r0   )r   bytesr   	ImmutablemutablejsonJSONEncoderdefault)r$   objr   r   r   r   x  s
   

zCustomEncoder.defaultN)r   r   r   r   r   r   r   r   r   w  s    r   c                 C   s.   |  d}|d ddd |dd D  S )z"Convert a snake str to camel case._r    c                 s   s    | ]}|  V  qd S r0   )title)r   xr   r   r   	<genexpr>  s    z to_camel_case.<locals>.<genexpr>ra   N)r   join)	snake_str
componentsr   r   r   to_camel_case  s   
$r   c                 C   s   i }|   D ]=\}}t|trt||t|< qt|tr=g }|D ]}t|tr0|t| q!|| q!||t|< q||t|< q|S )z3Recursive convert all keys in dict to google style.)itemsr   dictto_google_styler   listr   )dnew_dictkvnew_listir   r   r   r     s   


r   c                    s8    fdd t jjj| fddi|}r |S |S )z(Convert protobuf message to Python dict.c                    s   |   D ]C\}}t|tr || |< t|tr6g }|D ]}t|tr+| | q|| q|| |< q|v rCtt|| |< q|| |< q| S r0   )r   r   r   r   r   r   r   )r   r   r   r   r   _decode_keysr   r   r   r     s   




z%message_to_dict.<locals>._decode_keysuse_integers_for_enumsF)r   _privateprotobuf_compatr   )r   r   re   r   r   r   r   r     s   r   c                   @   s    e Zd ZdZdd Zdd ZdS )BunchzA dict with attribute-access.c                 C   s$   z|  |W S  ty   t|w r0   )__getitem__KeyErrorAttributeErrorr$   keyr   r   r   __getattr__  s
   zBunch.__getattr__c                 C   s   |  || d S r0   )__setitem__)r$   r   valuer   r   r   __setattr__     zBunch.__setattr__N)r   r   r   rn   r   r   r   r   r   r   r     s    r   c                 C   s   t d| jj )Nz%r objects are immutable)	TypeError	__class__r   r2   r   r   r   is_immutable  r   r   Tc                 C   sF   t | }|tu rt| S |tu rt| S |r!|tvr!td|| S )NzType {} can't be immutable.)typer   ImmutableDictr   ImmutableList_json_compatible_typesr   format)r   strict
value_typer   r   r   make_immutable  s   r   c                   @   s   e Zd Zedd ZdS )r   c                 C   s   d S r0   r   r2   r   r   r   r     s   zImmutable.mutableN)r   r   r   r   r   r   r   r   r   r     s    r   )	metaclassc                   @   s\   e Zd ZdZdZdd Zdd Zdd Zd	d
 Zdd Z	dd Z
dd Zdd Zdd ZdS )r   z Makes a :class:`list` immutable.)_list_proxyc                 C   sN   t |ttfvrtt | dt|tr| }|| _d gt| | _d S )Nz object is not a list.)	r   r   r   r   r   r   r   lenr   )r$   
list_valuer   r   r   r&     s   
zImmutableList.__init__c                 C      t | | jffS r0   )r   r   r$   protocolr   r   r   __reduce_ex__  r   zImmutableList.__reduce_ex__c                 C      | j S r0   )r   r2   r   r   r   r        zImmutableList.mutablec                 C       t |tr	| }t| j|S r0   )r   r   r   r   __eq__r   r$   otherr   r   r   r   	     
zImmutableList.__eq__c                 C   r   r0   )r   r   r   r   __ne__r   r   r   r   r   r     r   zImmutableList.__ne__c                 C   r   r0   )r   r   r   r   __contains__r   r$   itemr   r   r   r     r   zImmutableList.__contains__c                 C   s.   | j | }|d u rt| j|  }| j |< |S r0   )r   r   r   r$   r   proxyr   r   r   r     s   
zImmutableList.__getitem__c                 C   
   t | jS r0   )r   r   r2   r   r   r   __len__     
zImmutableList.__len__c                 C      d| j jt| jf S Nz%s(%s))r   r   r   __repr__r   r2   r   r   r   r   !     zImmutableList.__repr__N)r   r   r   rn   	__slots__r&   r   r   r   r   r   r   r   r   r   r   r   r   r     s    r   c                   @   st   e Zd ZdZdZdd Zdd Zdd Zdd
dZdd Z	dd Z
dd Zdd ZdefddZdd Zdd Zd	S )r   z Makes a :class:`dict` immutable.)_dictr   c                 C   sD   t |ttfvrtt | dt|tr| }|| _i | _d S )Nz object is not a dict.)r   r   r   r   r   r   r  r   )r$   
dict_valuer   r   r   r&   *  s   

zImmutableDict.__init__c                 C   r   r0   )r   r  r   r   r   r   r   2  r   zImmutableDict.__reduce_ex__c                 C   r   r0   )r  r2   r   r   r   r   5  r   zImmutableDict.mutableNc                 C   s&   z| | W S  t y   t| Y S w r0   )r   r   )r$   r   r   r   r   r   get8  s
   
zImmutableDict.getc                 C   r   r0   )r   r   r   r   r   r  r   r   r   r   r   >  r   zImmutableDict.__eq__c                 C   r   r0   )r   r   r   r   r   r  r   r   r   r   r   C  r   zImmutableDict.__ne__c                 C   r   r0   )r   r   r   r   r   r  r   r   r   r   r   H  r   zImmutableDict.__contains__c                 C   s2   | j |d }|d u rt| j|  }| j |< |S r0   )r   r  r   r  r   r   r   r   r   M  s   zImmutableDict.__getitem__r   c                 C   r   r0   )r   r  r2   r   r   r   r   S  r   zImmutableDict.__len__c                 C   sL   t | jt | jkr!| j | j  D ]}t| j| | j|< qt| jS r0   )r   r   r  keysr   iterr   r   r   r   __iter__V  s   
zImmutableDict.__iter__c                 C   r   r   )r   r   r   r   r  r2   r   r   r   r   \  r   zImmutableDict.__repr__r0   )r   r   r   rn   r  r&   r   r   r  r   r   r   r   rD   r   r  r   r   r   r   r   r   %  s    
r   Fc                    s    fdd}|S )Nc                    s   t   fdd}|S )Nc               
      s   	 z| i |I d H  W n: t jy7 } z r$td d |td d W Y d }~nd }~w tyH   td d Y nw t I d H  q)NTz-An async loop forever coroutine is cancelled rw   z0Can not cancel the async loop forever coroutine zError looping coroutine )asyncioCancelledErrorr]   rx   	exception	Exceptionsleep)rd   re   ex)cancellablecorointerval_secondsr   r   _looperg  s*   
z5async_loop_forever.<locals>._wrapper.<locals>._looperrj   )r  r  r  r  )r  r   _wrapperf  s   z$async_loop_forever.<locals>._wrapperr   )r  r  r  r   r  r   async_loop_forevere  s   r  rP   c                 C   s<   t j| d}|j}W d   n1 sw   Y  d| S )zConvert a Ray Client address of a running Ray cluster to its API server URL.

    Args:
        address: The Ray Client address, e.g. "ray://my-cluster".

    Returns:
        str: The API server URL of the cluster, e.g. "http://<head-node-ip>:8265".
    rP   Nhttp://)r   initdashboard_url)rP   client_contextr  r   r   r   $ray_client_address_to_api_server_url  s   	
r  c                 C   s^   t | } t| d}tjj| tjjj	|t
jt
jdd}|du r&tdd|  }|S )a  Parse a Ray cluster address into API server URL.

    When an address is provided, it will be used to query GCS for
    API server address from GCS, so a Ray cluster must be running.

    When an address is not provided, it will first try to auto-detect
    a running Ray instance, or look for local GCS process.

    Args:
        address: Ray cluster bootstrap address or Ray Client address.
            Could also be `auto`.

    Returns:
        API server HTTP URL.
    r     )	namespacenum_retriesNzCouldn't obtain the API server address from GCS. It is likely that the GCS server is down. Check gcs_server.[out | err] to see if it is still alive.r  )services%canonicalize_bootstrap_address_or_dier   r   experimentalrR   rT   r   utilsinternal_kv_get_with_retryray_constantsDASHBOARD_ADDRESSKV_NAMESPACE_DASHBOARD
ValueErrordecode)rP   rU   api_server_urlr   r   r   ray_address_to_api_server_url  s   

r)  c                 C   s   t jdrtdt jd   t jd } | r4d| v r4t| \}}|dkr3td|  d t| } nt| } td|  d | S )	a  Get Ray API server address from Ray bootstrap or Client address.

    If None, it will try to auto-detect a running Ray instance, or look
    for local GCS process.

    `address` is always overridden by the RAY_ADDRESS environment
    variable, just like the `address` argument in `ray.init()`.

    Args:
        address: Ray cluster bootstrap address or Ray Client address.
            Could also be "auto".

    Returns:
        API server HTTP URL, e.g. "http://<head-node-ip>:8265".
    RAY_ADDRESSzUsing RAY_ADDRESS=z://r   z6Retrieving API server address from Ray Client address z...zUsing API server address rw   )osenvironr  r]   debugr   r  r)  )rP   module_stringr   r   r   r   !get_address_for_submission_client  s   

r/  death_reasondeath_reason_messagec                 C   s^   | dkrd}n| dkrd}n| dkrd}n	| dkrd}nd	}|r-|r+|d
| 7 }|S |}|S )aE  Compose node state message based on death information.

    Args:
        death_reason: The reason of node death.
            This is a string representation of `gcs_pb2.NodeDeathInfo.Reason`.
        death_reason_message: The message of node death.
            This corresponds to `gcs_pb2.NodeDeathInfo.ReasonMessage`.
    EXPECTED_TERMINATIONzExpected terminationUNEXPECTED_TERMINATIONzUnexpected terminationAUTOSCALER_DRAIN_PREEMPTEDzTerminated due to preemptionAUTOSCALER_DRAIN_IDLEz(Terminated due to idle (no Ray activity)Nz: r   )r0  r1  state_messager   r   r   compose_state_message  s   r7  c                 C   s   | j D ]}|  qd S r0   )handlersclose)logger_instancehandlerr   r   r   close_logger_file_descriptor  s   

r<  r0   )T)F)cr4   r  r   rk   r   r   ro   r+  rz   r   r   base64r   collections.abcr   r   dataclassesr   enumr   typingr	   r
   r   r   r   ray._common.utilsr   #ray.core.generated.node_manager_pb2r   packaging.versionr   r   ray._private.protobuf_compatray._private.ray_constantsr   r#  ray._private.servicesr  ray.experimental.internal_kvr   rR   r   ray._private.gcs_utilsr   ray._private.utilsr   r   ray._rayletr   create_taskr   ensure_future	getLoggerr   r]   r   OSErrorr    ABCr!   r8   rE   r[   rv   r   r   r   rC   r   r   r   r   r   r   r   r   r   r   rD   floatrA   r   r   r   r   r   r   r   r   r   immutable_typeaddr  r  r)  r/  r7  r<  r   r   r   r   <module>   s    


eT
)



/<
("
