o
    biH                     @   s\  d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
mZmZ d dlZd dlm  mZ d dlm  mZ d dlm  mZ d dlmZ d dlmZ d dlmZ d dlmZm Z  d d	l!m"Z" d d
lm#Z#m$Z$ d dl%m&Z& d dlm'Z'm(Z(m)Z) d dl*Z*zd dl+Z+W n e,y   dZ+Y nw erd dl-m.Z. e/e0Z1eddZ2G dd dZ3dS )    N)ThreadPoolExecutor)Path)TYPE_CHECKINGListOptionalSetTuple)ray_constants)enable_monitor_loop_lag)env_integer)TagKeyrecord_extra_usage_tag)	GcsClient)%AVAILABLE_COMPONENT_NAMES_FOR_METRICSDASHBOARD_METRIC_PORT)DashboardPrometheusMetrics)DashboardHeadModuleDashboardHeadModuleConfigasync_loop_foreverSubprocessModuleHandle,RAY_DASHBOARD_DASHBOARD_HEAD_TPE_MAX_WORKERS   c                %   @   sV  e Zd Z	d0dededededededed	ed
ededededededededeee  f"ddZde	e
 de	d fddZedd Zeejdd Z	d0deee  dee	e
 e	d f fdd Z	d0deee  de	e
 fd!d"Z	d0deee  de	d fd#d$Zd%d& Zeejde	d fd'd(Z	)d1d*ejd+efd,d-Zd.d/ ZdS )2DashboardHeadN	http_host	http_porthttp_port_retriesgcs_addresscluster_id_hexnode_ip_addresslog_dirlogging_levellogging_formatlogging_filenamelogging_rotate_byteslogging_rotate_backup_counttemp_dirsession_dirminimalserve_frontendmodules_to_loadc                 C   s   || _ || _| j rd| _|dkrdn|| _|| _|| _|| _d| _d| _tt	dd| _
|dus1J || _|| _|| _|| _|	| _|
| _|| _|| _|| _|| _t|j| _d| _d| _|| _t | _t | _ d| _!dS )a  
        Args:
            http_host: The host address for the Http server.
            http_port: The port for the Http server.
            http_port_retries: The maximum retry to bind ports for the Http server.
            gcs_address: The GCS address in the {address}:{port} format.
            log_dir: The log directory. E.g., /tmp/session_latest/logs.
            logging_level: The logging level (e.g. logging.INFO, logging.DEBUG)
            logging_format: The format string for log messages
            logging_filename: The name of the log file
            logging_rotate_bytes: Max size in bytes before rotating log file
            logging_rotate_backup_count: Number of backup files to keep when rotating
            temp_dir: The temp directory. E.g., /tmp.
            session_dir: The session directory. E.g., tmp/session_latest.
            minimal: Whether or not it will load the minimal modules.
            serve_frontend: If configured, frontend HTML is
                served from the dashboard.
            modules_to_load: A set of module name in string to load.
                By default (None), it loads all available modules.
                Note that available modules could be changed depending on
                minimal flags.
        F	localhost	127.0.0.1Ndashboard_head_executor)max_workersthread_name_prefix)"r(   r)   r   r   r   _modules_to_load_modules_loadedmetricsr   r   	_executorr   r   r    r!   r"   r#   r$   r%   r&   r'   r   namesession_namegcs_error_subscribergcs_log_subscriberiposgetpidpidpsutilProcessdashboard_prochttp_server)selfr   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*    rA   F/home/ubuntu/.local/lib/python3.10/site-packages/ray/dashboard/head.py__init__1   s@   *


zDashboardHead.__init__dashboard_head_modulessubprocess_module_handlesr   c                    sJ   ddl m} || j| j| j| j| j| j| j| _	| j	
||I d H  d S )Nr   )HttpServerDashboardHead)ray.dashboard.http_server_headrF   r8   r   r   r   r   r5   r2   r?   run)r@   rD   rE   rF   rA   rA   rB   _configure_http_server   s   	z$DashboardHead._configure_http_serverc                 C   s&   | j s| jsd S | jsJ d| jjS )Nz+Accessing unsupported API in a minimal ray.)r1   r?   http_sessionr@   rA   rA   rB   rJ      s   zDashboardHead.http_sessionc                    sB   z| j jg d dI d H  W d S  ty    tjddd Y d S w )N)node_ipstimeoutz)Failed to check gcs aliveness, will retryT)exc_info)
gcs_clientasync_check_alive	ExceptionloggerwarningrK   rA   rA   rB   _gcs_check_alive   s   zDashboardHead._gcs_check_alivereturnc                 C   s   |  |}| |}dd |D dd |D B }t|t|t| ks(J d|dur;||kr;J d| d| d	d
| _||fS )z
        If minimal, only load DashboardHeadModule.
        If non-minimal, load both kinds of modules: DashboardHeadModule, SubprocessModule.

        If modules_to_load is not None, only load the modules in the set.
        c                 S   s   h | ]}t |jqS rA   )type__name__).0mrA   rA   rB   	<setcomp>   s    z.DashboardHead._load_modules.<locals>.<setcomp>c                 S   s   h | ]}|j jqS rA   )
module_clsrW   )rX   hrA   rA   rB   rZ      s    zmDuplicate module names. A module name can't be a DashboardHeadModule and a SubprocessModule at the same time.NFzActual loaded modules z/, doesn't match the requested modules to load, .T)_load_dashboard_head_modules_load_subprocess_module_handleslenr1   )r@   r*   rD   rE   	all_namesrA   rA   rB   _load_modules   s(   
	
zDashboardHead._load_modulesc                    s   g }t t}t| j| j| j| j| j| j	| j
| j| j| jd
} dur, fdd|D }td  d |D ]}tdtj d| d ||}|| q7td	t| d
| d |S )zLoad `DashboardHeadModule`s.

        Args:
            modules: A list of module names to load. By default (None),
                it loads all modules.
        )
r(   r   r5   r   r    r&   r'   r8   r   r   Nc                       g | ]	}|j  v r|qS rA   rW   rX   clsr*   rA   rB   
<listcomp>       z>DashboardHead._load_dashboard_head_modules.<locals>.<listcomp>zDashboardHeadModules to load: r]   Loading : Loaded z dashboard head modules: )dashboard_utilsget_all_modulesr   r   r(   r   r5   r   r    r&   r'   r8   r   r   rR   inforW   appendr`   )r@   r*   moduleshead_cls_listconfigrf   crA   rg   rB   r^      s2   	

z*DashboardHead._load_dashboard_head_modulesc                    s  | j r
td g S ddlm} ddlm}m} g }t	|}t
jj }|| j| j| j| j| j| j| j| j| j| j| jtt| jd d} durT fdd	|D }|D ]}	td
|j d|	 d |||	|}
||
 qVtdt| d| d |S )z
        If minimal, return an empty list.
        If non-minimal, load `SubprocessModule`s by creating Handles to them.

        Args:
            modules: A list of module names to load. By default (None),
                it loads all modules.
        z.Subprocess modules not loaded in minimal mode.r   r   )SubprocessModuleSubprocessModuleConfigsockets)r   r   r5   r&   r'   r!   r"   r    r#   r$   r%   
socket_dirNc                    rc   rA   rd   re   rg   rA   rB   rh     ri   zADashboardHead._load_subprocess_module_handles.<locals>.<listcomp>rj   rk   r]   rl   z subprocess modules: )r(   rR   ro   !ray.dashboard.subprocesses.handler   !ray.dashboard.subprocesses.moduleru   rv   rm   rn   ray_commonutilsget_or_create_event_loopr   r   r5   r&   r'   r!   r"   r    r#   r$   r%   strr   rW   rp   r`   )r@   r*   r   ru   rv   handlessubprocess_cls_listlooprs   rf   handlerA   rg   rB   r_      s@   


z-DashboardHead._load_subprocess_module_handlesc                    s   t  }t s
J |d usJ | j dt }|jd | dd dI d H  tr\z"t	d
t | jdkr<ddini }tjdt|jd| W |S  ty[   td	 Y |S w tsctd
 |S )N:DashboardMetricsAddressT	namespacez,Starting dashboard metrics server on port {}r,   addr)portregistryz8An exception occurred while starting the metrics server.z?`prometheus_client` not found, so metrics will not be exported.rA   )r   internal_kv_internal_kv_initializedr8   r   async_internal_kv_putencodeprometheus_clientrR   ro   formatstart_http_serverr   rQ   	exceptionrS   )r@   rO   r2   addresskwargsrA   rA   rB   _setup_metrics*  sD   zDashboardHead._setup_metricsc                    s   | j | jtjd| jd}dtv sJ | | j |D ]}|jd us$J t	
|jj}| ||jj qtjj }| jjjdi |tt| | jd urf| jjjdi |t| j d | _d S d S )N	dashboardr8   r;   Version	ComponentSessionNamerA   )r8   r;   r{   __version__r5   r    _record_cpu_mem_metrics_for_procr>   processr<   r=   r[   rW   r|   r}   r~   r2   metrics_event_loop_taskslabelssetr`   asyncio	all_tasks_event_loop_lag_s_maxmetrics_event_loop_lagfloat)r@   rE   r   subprocess_module_handleprocr   rA   rA   rB   _record_dashboard_metricsL  s2   

z'DashboardHead._record_dashboard_metrics r   module_namec                 C   s   | j |jtj|s
dnd| | jd}|jddgd}| jjjd	i |	t
|dd |dd ur`| jjjd	i |	t
|djd  | jjjd	i |	t
|djd  d S d S )
Nr   
dashboard_r   cpu_percentmemory_full_info)attrsg        g    .ArA   )r8   r;   r{   r   r5   as_dictr2   metrics_dashboard_cpur   r   r   getmetrics_dashboard_mem_ussussmetrics_dashboard_mem_rssrss)r@   r   r   r   
proc_attrsrA   rA   rB   r   m  s$   z.DashboardHead._record_cpu_mem_metrics_for_procc              
      s   j }t| jd _t j   j\}}|D ]}|  q|D ]}|	  q% j
sO  jI d H  _d  _ fdd}t| t | _zt sVJ ttjd W n tyw } ztd|  W Y d }~nd }~ww  j j}} jrtd  ||I d H   j  \}}td| d|  ntd	  jt!j"kr j#n|}	 jj$t!j%& |	 d| & d
t!j'd  ( g}
|D ]	}|
)|*  qtj+|
 I d H   jr j, I d H  d S d S )N)r   
cluster_idc                    s   t  jpd|  _d S )Nr   )maxr   )lag_srK   rA   rB   
on_new_lag  s   z%DashboardHead.run.<locals>.on_new_lagFalsez`Failed to record the dashboard usage. This error message is harmless and can be ignored. Error: zInitialize the http server.zhttp server initialized at r   zhttp server disabled.Tr   )-r   r   r   rO   r   _initialize_internal_kvrb   r0   start_modulewait_for_module_readyr(   r   r2   r   r
   r   create_taskr   record_dashboard_metrics_taskr   r   r   DASHBOARD_USEDrQ   rR   rS   r   r   r)   ro   rI   r?   get_addressr	   DEFAULT_DASHBOARD_IPr8   internal_kv_putDASHBOARD_ADDRESSr   KV_NAMESPACE_DASHBOARDrT   rp   rH   gathercleanup)r@   r   rD   rE   r   r   er   r   dashboard_http_hostconcurrent_tasksrY   rA   rK   rB   rH     sr   




zDashboardHead.run)N)r   )rW   
__module____qualname__r   intboolr   r   rC   r   r   rI   propertyrJ   r   dashboard_consts GCS_CHECK_ALIVE_INTERVAL_SECONDSrT   r   rb   r^   r_   r   rm   METRICS_RECORD_INTERVAL_Sr   r<   r=   r   rH   rA   rA   rA   rB   r   0   s    	


S


	

 

*

6
"!
r   )4r   loggingr9   concurrent.futuresr   pathlibr   typingr   r   r   r   r   r{   ray.dashboard.constsr   constsr   ray.dashboard.utilsr}   rm   ray.experimental.internal_kvexperimentalr   ray._privater	   ray._private.async_utilsr
   ray._private.ray_constantsr   ray._private.usage.usage_libr   r   ray._rayletr   r   r   ray.dashboard.dashboard_metricsr   r   r   r   r<   r   ImportErrorry   r   	getLoggerrW   rR   r   r   rA   rA   rA   rB   <module>   s>    
