o
    $icI                     @   sh  d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
mZmZ d dlZd dlm  mZ d dlm  mZ d dlm  mZ d dlmZ d dlmZmZ d dlmZ d dlm Z  d d	l!m"Z" d d
l#m$Z$ d dlm%Z%m&Z& d dl'm(Z( d dlm)Z)m*Z*m+Z+ d dl,Z,zd dl-Z-W n e.y   dZ-Y nw erd dl/m0Z0 e1e2Z3e"ddZ4G dd dZ5dS )    N)ThreadPoolExecutor)Path)TYPE_CHECKINGListOptionalSetTuple)build_address)TagKeyrecord_extra_usage_tag)ray_constants)enable_monitor_loop_lag)env_integer)	GcsClient)%AVAILABLE_COMPONENT_NAMES_FOR_METRICSDASHBOARD_METRIC_PORT)DashboardPrometheusMetrics)DashboardHeadModuleDashboardHeadModuleConfigasync_loop_foreverSubprocessModuleHandle,RAY_DASHBOARD_DASHBOARD_HEAD_TPE_MAX_WORKERS   c                %   @   sV  e Zd Z	d0dededededededed	ed
ededededededededeee  f"ddZde	e
 de	d fddZedd Zeejdd Z	d0deee  dee	e
 e	d f fdd Z	d0deee  de	e
 fd!d"Z	d0deee  de	d fd#d$Zd%d& Zeejde	d fd'd(Z	)d1d*ejd+efd,d-Zd.d/ ZdS )2DashboardHeadN	http_host	http_porthttp_port_retriesgcs_addresscluster_id_hexnode_ip_addresslog_dirlogging_levellogging_formatlogging_filenamelogging_rotate_byteslogging_rotate_backup_counttemp_dirsession_dirminimalserve_frontendmodules_to_loadc                 C   s   || _ || _| j rd| _|dkrdn|| _|| _|| _|| _d| _d| _tt	dd| _
|dus1J || _|| _|| _|| _|	| _|
| _|| _|| _|| _|| _t|j| _d| _d| _|| _t | _t | _ d| _!dS )a  
        Args:
            http_host: The host address for the Http server.
            http_port: The port for the Http server.
            http_port_retries: The maximum retry to bind ports for the Http server.
            gcs_address: The GCS address in the {address}:{port} format.
            log_dir: The log directory. E.g., /tmp/session_latest/logs.
            logging_level: The logging level (e.g. logging.INFO, logging.DEBUG)
            logging_format: The format string for log messages
            logging_filename: The name of the log file
            logging_rotate_bytes: Max size in bytes before rotating log file
            logging_rotate_backup_count: Number of backup files to keep when rotating
            temp_dir: The temp directory. E.g., /tmp.
            session_dir: The session directory. E.g., tmp/session_latest.
            minimal: Whether or not it will load the minimal modules.
            serve_frontend: If configured, frontend HTML is
                served from the dashboard.
            modules_to_load: A set of module name in string to load.
                By default (None), it loads all available modules.
                Note that available modules could be changed depending on
                minimal flags.
        F	localhost	127.0.0.1Ndashboard_head_executor)max_workersthread_name_prefix)"r)   r*   r   r   r   _modules_to_load_modules_loadedmetricsr   r   	_executorr   r   r!   r"   r#   r$   r%   r&   r'   r(   r   namesession_namegcs_error_subscribergcs_log_subscriberiposgetpidpidpsutilProcessdashboard_prochttp_server)selfr   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+    rB   O/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/ray/dashboard/head.py__init__2   s@   *


zDashboardHead.__init__dashboard_head_modulessubprocess_module_handlesr   c                    sJ   ddl m} || j| j| j| j| j| j| j| _	| j	
||I d H  d S )Nr   )HttpServerDashboardHead)ray.dashboard.http_server_headrG   r9   r   r   r   r   r6   r3   r@   run)rA   rE   rF   rG   rB   rB   rC   _configure_http_server   s   	z$DashboardHead._configure_http_serverc                 C   s&   | j s| jsd S | jsJ d| jjS )Nz+Accessing unsupported API in a minimal ray.)r2   r@   http_sessionrA   rB   rB   rC   rK      s   zDashboardHead.http_sessionc                    sB   z| j jg d dI d H  W d S  ty    tjddd Y d S w )N)node_idstimeoutz)Failed to check gcs aliveness, will retryT)exc_info)
gcs_clientasync_check_alive	ExceptionloggerwarningrL   rB   rB   rC   _gcs_check_alive   s   zDashboardHead._gcs_check_alivereturnc                 C   s   |  |}| |}dd |D dd |D B }t|t|t| ks(J d|dur;||kr;J d| d| d	d
| _||fS )z
        If minimal, only load DashboardHeadModule.
        If non-minimal, load both kinds of modules: DashboardHeadModule, SubprocessModule.

        If modules_to_load is not None, only load the modules in the set.
        c                 S   s   h | ]}t |jqS rB   )type__name__).0mrB   rB   rC   	<setcomp>   s    z.DashboardHead._load_modules.<locals>.<setcomp>c                 S   s   h | ]}|j jqS rB   )
module_clsrX   )rY   hrB   rB   rC   r[      s    zmDuplicate module names. A module name can't be a DashboardHeadModule and a SubprocessModule at the same time.NFzActual loaded modules z/, doesn't match the requested modules to load, .T)_load_dashboard_head_modules_load_subprocess_module_handleslenr2   )rA   r+   rE   rF   	all_namesrB   rB   rC   _load_modules   s(   
	
zDashboardHead._load_modulesc                    s   g }t t}t| j| j| j| j| j| j	| j
| j| j| jd
} dur, fdd|D }td  d |D ]}tdtj d| d ||}|| q7td	t| d
| d |S )zLoad `DashboardHeadModule`s.

        Args:
            modules: A list of module names to load. By default (None),
                it loads all modules.
        )
r)   r   r6   r   r!   r'   r(   r9   r   r   Nc                       g | ]	}|j  v r|qS rB   rX   rY   clsr+   rB   rC   
<listcomp>       z>DashboardHead._load_dashboard_head_modules.<locals>.<listcomp>zDashboardHeadModules to load: r^   Loading : Loaded z dashboard head modules: )dashboard_utilsget_all_modulesr   r   r)   r   r6   r   r!   r'   r(   r9   r   r   rS   inforX   appendra   )rA   r+   moduleshead_cls_listconfigrg   crB   rh   rC   r_      s2   	

z*DashboardHead._load_dashboard_head_modulesc                    s  | j r
td g S ddlm} ddlm}m} g }t	|}t
jj }|| j| j| j| j| j| j| j| j| j| j| jtt| jd d} durT fdd	|D }|D ]}	td
|j d|	 d |||	|}
||
 qVtdt| d| d |S )z
        If minimal, return an empty list.
        If non-minimal, load `SubprocessModule`s by creating Handles to them.

        Args:
            modules: A list of module names to load. By default (None),
                it loads all modules.
        z.Subprocess modules not loaded in minimal mode.r   r   )SubprocessModuleSubprocessModuleConfigsockets)r   r   r6   r'   r(   r"   r#   r!   r$   r%   r&   
socket_dirNc                    rd   rB   re   rf   rh   rB   rC   ri     rj   zADashboardHead._load_subprocess_module_handles.<locals>.<listcomp>rk   rl   r^   rm   z subprocess modules: )r)   rS   rp   !ray.dashboard.subprocesses.handler   !ray.dashboard.subprocesses.modulerv   rw   rn   ro   ray_commonutilsget_or_create_event_loopr   r   r6   r'   r(   r"   r#   r!   r$   r%   r&   strr   rX   rq   ra   )rA   r+   r   rv   rw   handlessubprocess_cls_listlooprt   rg   handlerB   rh   rC   r`      s@   


z-DashboardHead._load_subprocess_module_handlesc                    s   t  }t s
J |d usJ t| jt}|jd | dd dI d H  trZz"t	
dt | jdkr:ddini }tjd
t|jd| W |S  tyY   t	d Y |S w tsat	d	 |S )NDashboardMetricsAddressT	namespacez,Starting dashboard metrics server on port {}r-   addr)portregistryz8An exception occurred while starting the metrics server.z?`prometheus_client` not found, so metrics will not be exported.rB   )r   internal_kv_internal_kv_initializedr	   r9   r   async_internal_kv_putencodeprometheus_clientrS   rp   formatstart_http_serverr   rR   	exceptionrT   )rA   rP   r3   addresskwargsrB   rB   rC   _setup_metrics+  sD   zDashboardHead._setup_metricsc                    s   | j | jtjd| jd}dtv sJ | | j |D ]}|jd us$J t	
|jj}| ||jj qtjj }| jjjdi |tt| | jd urf| jjjdi |t| j d | _d S d S )N	dashboardr9   r<   Version	ComponentSessionNamerB   )r9   r<   r|   __version__r6   r    _record_cpu_mem_metrics_for_procr?   processr=   r>   r\   rX   r}   r~   r   r3   metrics_event_loop_taskslabelssetra   asyncio	all_tasks_event_loop_lag_s_maxmetrics_event_loop_lagfloat)rA   rF   r   subprocess_module_handleprocr   rB   rB   rC   _record_dashboard_metricsM  s2   

z'DashboardHead._record_dashboard_metrics r   module_namec                 C   s   | j |jtj|s
dnd| | jd}|jddgd}| jjjd	i |	t
|dd |dd ur`| jjjd	i |	t
|djd  | jjjd	i |	t
|djd  d S d S )
Nr   
dashboard_r   cpu_percentmemory_full_info)attrsg        g    .ArB   )r9   r<   r|   r   r6   as_dictr3   metrics_dashboard_cpur   r   r   getmetrics_dashboard_mem_ussussmetrics_dashboard_mem_rssrss)rA   r   r   r   
proc_attrsrB   rB   rC   r   n  s$   z.DashboardHead._record_cpu_mem_metrics_for_procc              
      s   j }t| jd _t j   j\}}|D ]}|  q|D ]}|	  q% j
sO  jI d H  _d  _ fdd}t| t | _zt sVJ ttjd W n tyw } ztd|  W Y d }~nd }~ww  j j}} jrtd  ||I d H   j  \}}tdt!||  ntd  jt"j#kr j$n|}	 jj%t"j&' t!|	|' d	t"j(d
  ) g}
|D ]	}|
*|+  qtj,|
 I d H   jr j- I d H  d S d S )N)r   
cluster_idc                    s   t  jpd|  _d S )Nr   )maxr   )lag_srL   rB   rC   
on_new_lag  s   z%DashboardHead.run.<locals>.on_new_lagFalsez`Failed to record the dashboard usage. This error message is harmless and can be ignored. Error: zInitialize the http server.zhttp server initialized at zhttp server disabled.Tr   ).r   r   r   rP   r   _initialize_internal_kvrc   r1   start_modulewait_for_module_readyr)   r   r3   r   r   r   create_taskr   record_dashboard_metrics_taskr   r   r
   DASHBOARD_USEDrR   rS   rT   r   r   r*   rp   rJ   r@   get_addressr	   r   DEFAULT_DASHBOARD_IPr9   internal_kv_putDASHBOARD_ADDRESSr   KV_NAMESPACE_DASHBOARDrU   rq   rI   gathercleanup)rA   r   rE   rF   r   r   er   r   dashboard_http_hostconcurrent_tasksrZ   rB   rL   rC   rI     sv   




zDashboardHead.run)N)r   )rX   
__module____qualname__r   intboolr   r   rD   r   r   rJ   propertyrK   r   dashboard_consts GCS_CHECK_ALIVE_INTERVAL_SECONDSrU   r   rc   r_   r`   r   rn   METRICS_RECORD_INTERVAL_Sr   r=   r>   r   rI   rB   rB   rB   rC   r   1   s    	


S


	

 

*

6
"!
r   )6r   loggingr:   concurrent.futuresr   pathlibr   typingr   r   r   r   r   r|   ray.dashboard.constsr   constsr   ray.dashboard.utilsr~   rn   ray.experimental.internal_kvexperimentalr   ray._common.network_utilsr	   ray._common.usage.usage_libr
   r   ray._privater   ray._private.async_utilsr   ray._private.ray_constantsr   ray._rayletr   r   r   ray.dashboard.dashboard_metricsr   r   r   r   r=   r   ImportErrorrz   r   	getLoggerrX   rS   r   r   rB   rB   rB   rC   <module>   s@    
