o
    $i]?                     @   s>  d dl Z d dlZd dlZd dlZd dlmZ d dlZd dlm  m	Z
 d dlm  mZ d dlmZmZ d dlmZmZmZmZmZmZ d dlmZmZmZmZ d dlmZ d dlm Z! e"e#Z$e$%ej& dZ'd	Z(d
Z)dZ*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dd Z8G dd de9Z:G dd deZ;dS )    N)quote)!PROMETHEUS_SERVICE_DISCOVERY_FILESESSION_LATEST)generate_data_grafana_dashboard"generate_default_grafana_dashboard+generate_serve_deployment_grafana_dashboard generate_serve_grafana_dashboard$generate_serve_llm_grafana_dashboard generate_train_grafana_dashboard)DASHBOARD_PROVISIONING_TEMPLATEGRAFANA_DATASOURCE_TEMPLATEGRAFANA_INI_TEMPLATEPROMETHEUS_YML_TEMPLATE)SubprocessModule)SubprocessRouteTableRAY_METRICS_OUTPUT_ROOTzhttp://localhost:9090RAY_PROMETHEUS_HOSTz{}RAY_PROMETHEUS_HEADERS
PrometheusRAY_PROMETHEUS_NAMEz	-/healthyzhttp://localhost:3000RAY_GRAFANA_HOSTRAY_GRAFANA_ORG_ID1RAY_GRAFANA_CLUSTER_FILTERDISABLEDRAY_GRAFANA_IFRAME_HOST(RAY_METRICS_GRAFANA_DASHBOARD_OUTPUT_DIRz
api/healthc                 C   sl   t | }t|trtdd | D r|S t|tr'tdd |D r'|S tt dd d d d )	Nc                 s   s(    | ]\}}t |tot |tV  qd S N
isinstancestr).0kv r$   g/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/ray/dashboard/modules/metrics/metrics_head.py	<genexpr>>   s   & z%parse_prom_headers.<locals>.<genexpr>c                 s   s.    | ]}t |d kotdd |D V  qdS )   c                 s   s    | ]}t |tV  qd S r   r   )r!   r#   r$   r$   r%   r&   A   s    z/parse_prom_headers.<locals>.<genexpr>.<genexpr>N)lenall)r!   er$   r$   r%   r&   A   s   , z0 should be a JSON string in one of the formats:
z11) An object with string keys and string values.
z:2) an array of string arrays with 2 string elements each.
z*For example, {"H1": "V1", "H2": "V2"} and
z5[["H1", "V1"], ["H2", "V2"], ["H2", "V3"]] are valid.)	jsonloadsr   dictr)   itemslist
ValueErrorPROMETHEUS_HEADERS_ENV_VAR)prometheus_headersparsedr$   r$   r%   parse_prom_headers;   s$   


r4   c                       s   e Zd Z fddZ  ZS )PrometheusQueryErrorc                    s$   d| d| | _ t | j  d S )Nz-Error fetching data from prometheus. status: z, message: )messagesuper__init__)selfstatusr6   	__class__r$   r%   r8   M   s   zPrometheusQueryError.__init__)__name__
__module____qualname__r8   __classcell__r$   r$   r;   r%   r5   L   s    r5   c                       sn   e Zd Z fddZeddejjfddZ	eddd	 Z
d
d Zdd Z fddZdd Z  ZS )MetricsHeadc                    s   t  j|i | tjtt| _tjtt	| _
tj| jd}ttjtt| _tj| jtd}tjt|| _tjt|| _tj| jd| _tj| jd| _tjttj| jd| _tjtt| _tjtt| _ tjt!| _"i | _#d S )Nmetricsgrafana
dashboards)$r7   r8   osenvirongetGRAFANA_HOST_ENV_VARDEFAULT_GRAFANA_HOSTgrafana_hostPROMETHEUS_HOST_ENV_VARDEFAULT_PROMETHEUS_HOSTprometheus_hostpathjoinsession_dirr4   r1   DEFAULT_PROMETHEUS_HEADERSr2   temp_dirr   METRICS_OUTPUT_ROOT_ENV_VAR_metrics_root_metrics_root_session_latest_grafana_config_output_path*_grafana_session_latest_config_output_path$GRAFANA_DASHBOARD_OUTPUT_DIR_ENV_VAR_grafana_dashboard_output_dirPROMETHEUS_NAME_ENV_VARDEFAULT_PROMETHEUS_NAME_prometheus_nameGRAFANA_ORG_ID_ENV_VARDEFAULT_GRAFANA_ORG_ID_grafana_org_idGRAFANA_CLUSTER_FILTER_ENV_VAR_grafana_cluster_filter_dashboard_uids)r9   argskwargsdefault_metrics_rootsession_latest_metrics_rootr;   r$   r%   r8   V   sJ   
zMetricsHead.__init__z/api/grafana_healthreturnc                    sz  | j tkrtjtjjdtdS tj	t
| j }| j  dt }zw| j	|4 I dH _}|jdkrGtjtjjd|jdW  d  I dH  W S | I dH }|d d	krktjtjjd
|j|dW  d  I dH  W S tjtjjd|| j| j| j| j| jdW  d  I dH  W S 1 I dH sw   Y  W dS  ty } ztjd|d tjtjjdt|dW  Y d}~S d}~ww )z<
        Endpoint that checks if Grafana is running
        zGrafana disabled)status_coder6   rJ   /N   zGrafana healthcheck failedrh   r6   r:   databaseokz,Grafana healthcheck failed. Database not ok.)rh   r6   r:   r+   zGrafana running)rh   r6   rJ   grafana_org_idsession_namedashboard_uidsdashboard_datasourcegrafana_cluster_filterz4Error fetching grafana endpoint. Is grafana running?exc_info)rh   r6   	exception)rJ   GRAFANA_HOST_DISABLED_VALUEdashboard_optional_utilsrest_responsedashboard_utilsHTTPStatusCodeOKrE   rF   rG   GRAFANA_IFRAME_HOST_ENV_VARGRAFANA_HEALTHCHECK_PATHhttp_sessionr:   INTERNAL_ERRORr+   r_   ro   rb   r\   ra   	Exceptionloggerdebugr    )r9   reqgrafana_iframe_hostrN   respr+   r*   r$   r$   r%   grafana_health   sf   

4zMetricsHead.grafana_healthz/api/prometheus_healthc              
      s   zS| j  dt }| jj|| jd4 I d H 0}|jdkr3tjtj	j
d|jdW  d   I d H  W S tjtj	jddW  d   I d H  W S 1 I d H sMw   Y  W d S  tyy } ztjd|d	 tjtj	j
dt|d
W  Y d }~S d }~ww )Nri   headersrj   zprometheus healthcheck failed.rk   zprometheus running)rh   r6   z:Error fetching prometheus endpoint. Is prometheus running?rs   )rh   r6   reason)rM   PROMETHEUS_HEALTHCHECK_PATHr~   rG   r2   r:   rw   rx   ry   rz   r   r{   r   r   r   r    )r9   r   rN   r   r*   r$   r$   r%   prometheus_health   s<   

4zMetricsHead.prometheus_healthc           
   
   C   s  t j| jrt| j t j| jdd t j| jd}t j| jd}t	t j| jdd}|
tj|d W d   n1 sDw   Y  t j|d}t j|dd t	t j|d	d}|
tj| jd
 W d   n1 svw   Y  t jtt}tt jtt}g }t|tr|}nt|trt| }t j|d}t j|dd t j| jdd t	t j|d	d"}|
t|| jdd t|D dd t|D d W d   n1 sw   Y  t	t j| jdd}t \}	| jd< |
|	 W d   n	1 sw   Y  t	t j| jdd}t \}	| jd< |
|	 W d   n	1 s7w   Y  t	t j| jdd}t  \}	| jd< |
|	 W d   n	1 s`w   Y  t	t j| jdd}t! \}	| jd< |
|	 W d   n	1 sw   Y  t	t j| jdd}t" \}	| jd< |
|	 W d   n	1 sw   Y  t	t j| jdd}t# \}	| jd< |
|	 W d   dS 1 sw   Y  dS )zY
        Creates the Grafana configurations that are by default provided by Ray.
        Texist_okprovisioningzgrafana.iniw)grafana_provisioning_folderNrD   zdefault.yml)dashboard_output_folderdatasourcesc                 S   s$   i | ]\}\}}d |d  |qS )httpHeaderName   r$   )r!   iheader_r$   r$   r%   
<dictcomp>,      
z?MetricsHead._create_default_grafana_configs.<locals>.<dictcomp>c                 S   s$   i | ]\}\}}d |d  |qS )httpHeaderValuer   r$   )r!   r   r   valuer$   r$   r%   r   0  r   )rM   prometheus_namejsonDatasecureJsonDatazdefault_grafana_dashboard.jsondefaultzserve_grafana_dashboard.jsonservez'serve_deployment_grafana_dashboard.jsonserve_deploymentz serve_llm_grafana_dashboard.json	serve_llmzdata_grafana_dashboard.jsondataztrain_grafana_dashboard.jsontrain)$rE   rN   existsrV   shutilrmtreemakedirsrO   rW   openwriter   formatr   rY   rF   rG   rK   rL   r4   r1   rQ   r   r/   r-   r.   r   r\   	enumerater   rb   r   r   r	   r   r
   )
r9   r   'grafana_prov_folder_with_latest_sessionfdashboard_provisioning_pathrM   r2   prometheus_header_pairsdata_sources_pathcontentr$   r$   r%   _create_default_grafana_configs   sL  

	$z+MetricsHead._create_default_grafana_configsc                 C   s   t j| jdd}t j|rt | t jt j|dd t j| jt	}t
|d}|tj|d W d   dS 1 sAw   Y  dS )z\
        Creates the Prometheus configurations that are by default provided by Ray.
        
prometheuszprometheus.ymlTr   r   )(prom_metrics_service_discovery_file_pathN)rE   rN   rO   rT   r   remover   dirnamerR   r   r   r   r   r   )r9   prometheus_config_output_pathprom_discovery_file_pathr   r$   r$   r%   "_create_default_prometheus_configs|  s    
"z.MetricsHead._create_default_prometheus_configsc                    s&   t   I d H  |   |   d S r   )r7   runr   r   )r9   r;   r$   r%   r     s   zMetricsHead.runc              	      s   | j j| j dt| | jd4 I d H &}|jdkr/| I d H }|W  d   I d H  S | I d H }t|j|1 I d H sBw   Y  d S )Nz/api/v1/query?query=r   rj   )	r~   rG   rM   r   r2   r:   r+   textr5   )r9   queryr   	prom_datar6   r$   r$   r%   _query_prometheus  s   
zMetricsHead._query_prometheus)r=   r>   r?   r8   routesrG   aiohttpwebResponser   r   r   r   r   r   r@   r$   r$   r;   r%   rA   U   s    *8
 'rA   )<r+   loggingrE   r   urllib.parser   r   ray.dashboard.optional_utils	dashboardoptional_utilsrw   ray.dashboard.utilsutilsry   ray._private.ray_constantsr   r   7ray.dashboard.modules.metrics.grafana_dashboard_factoryr   r   r   r   r	   r
   'ray.dashboard.modules.metrics.templatesr   r   r   r   !ray.dashboard.subprocesses.moduler   !ray.dashboard.subprocesses.routesr   r   	getLoggerr=   r   setLevelINFOrS   rL   rK   rQ   r1   r[   rZ   r   rI   rH   r]   r^   r`   rv   r|   rX   r}   r4   r   r5   rA   r$   r$   r$   r%   <module>   sF     
	