o
    bi}>                     @   s:  d dl Z d dlZd dlZd dlZd dlmZ d dlZd dlm  m	Z
 d dlm  mZ d dlmZmZ d dlmZmZmZmZmZmZ d dlmZmZmZmZ d dlmZ d dlm Z! e"e#Z$e$%ej& dZ'd	Z(d
Z)dZ*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dd Z7G dd de8Z9G dd deZ:dS )    N)quote)!PROMETHEUS_SERVICE_DISCOVERY_FILESESSION_LATEST)generate_data_grafana_dashboard"generate_default_grafana_dashboard+generate_serve_deployment_grafana_dashboard generate_serve_grafana_dashboard$generate_serve_llm_grafana_dashboard generate_train_grafana_dashboard)DASHBOARD_PROVISIONING_TEMPLATEGRAFANA_DATASOURCE_TEMPLATEGRAFANA_INI_TEMPLATEPROMETHEUS_YML_TEMPLATE)SubprocessModule)SubprocessRouteTableRAY_METRICS_OUTPUT_ROOTzhttp://localhost:9090RAY_PROMETHEUS_HOSTz{}RAY_PROMETHEUS_HEADERS
PrometheusRAY_PROMETHEUS_NAMEz	-/healthyzhttp://localhost:3000RAY_GRAFANA_HOSTRAY_GRAFANA_ORG_ID1DISABLEDRAY_GRAFANA_IFRAME_HOST(RAY_METRICS_GRAFANA_DASHBOARD_OUTPUT_DIRz
api/healthc                 C   sl   t | }t|trtdd | D r|S t|tr'tdd |D r'|S tt dd d d d )	Nc                 s   s(    | ]\}}t |tot |tV  qd S N
isinstancestr).0kv r#   ^/home/ubuntu/.local/lib/python3.10/site-packages/ray/dashboard/modules/metrics/metrics_head.py	<genexpr>=   s   & z%parse_prom_headers.<locals>.<genexpr>c                 s   s.    | ]}t |d kotdd |D V  qdS )   c                 s   s    | ]}t |tV  qd S r   r   )r    r"   r#   r#   r$   r%   @   s    z/parse_prom_headers.<locals>.<genexpr>.<genexpr>N)lenall)r    er#   r#   r$   r%   @   s   , z0 should be a JSON string in one of the formats:
z11) An object with string keys and string values.
z:2) an array of string arrays with 2 string elements each.
z*For example, {"H1": "V1", "H2": "V2"} and
z5[["H1", "V1"], ["H2", "V2"], ["H2", "V3"]] are valid.)	jsonloadsr   dictr(   itemslist
ValueErrorPROMETHEUS_HEADERS_ENV_VAR)prometheus_headersparsedr#   r#   r$   parse_prom_headers:   s$   


r3   c                       s   e Zd Z fddZ  ZS )PrometheusQueryErrorc                    s$   d| d| | _ t | j  d S )Nz-Error fetching data from prometheus. status: z, message: )messagesuper__init__)selfstatusr5   	__class__r#   r$   r7   L   s   zPrometheusQueryError.__init__)__name__
__module____qualname__r7   __classcell__r#   r#   r:   r$   r4   K   s    r4   c                       sn   e Zd Z fddZeddejjfddZ	eddd	 Z
d
d Zdd Z fddZdd Z  ZS )MetricsHeadc                    s   t  j|i | tjtt| _tjtt	| _
tj| jd}ttjtt| _tj| jtd}tjt|| _tjt|| _tj| jd| _tj| jd| _tjttj| jd| _tjtt| _tjtt| _ i | _!d S )Nmetricsgrafana
dashboards)"r6   r7   osenvirongetGRAFANA_HOST_ENV_VARDEFAULT_GRAFANA_HOSTgrafana_hostPROMETHEUS_HOST_ENV_VARDEFAULT_PROMETHEUS_HOSTprometheus_hostpathjoinsession_dirr3   r0   DEFAULT_PROMETHEUS_HEADERSr1   temp_dirr   METRICS_OUTPUT_ROOT_ENV_VAR_metrics_root_metrics_root_session_latest_grafana_config_output_path*_grafana_session_latest_config_output_path$GRAFANA_DASHBOARD_OUTPUT_DIR_ENV_VAR_grafana_dashboard_output_dirPROMETHEUS_NAME_ENV_VARDEFAULT_PROMETHEUS_NAME_prometheus_nameGRAFANA_ORG_ID_ENV_VARDEFAULT_GRAFANA_ORG_ID_grafana_org_id_dashboard_uids)r8   argskwargsdefault_metrics_rootsession_latest_metrics_rootr:   r#   r$   r7   U   sH   
zMetricsHead.__init__z/api/grafana_healthreturnc              
      sv  | j tkrtjtjjdtdS tj	t
| j }| j  dt }zu| j	|4 I dH ]}|jdkrGtjtjjd|jdW  d  I dH  W S | I dH }|d d	krktjtjjd
|j|dW  d  I dH  W S tjtjjd|| j| j| j| jdW  d  I dH  W S 1 I dH sw   Y  W dS  ty } ztjd|d tjtjjdt|dW  Y d}~S d}~ww )z<
        Endpoint that checks if Grafana is running
        zGrafana disabled)status_coder5   rI   /N   zGrafana healtcheck failedre   r5   r9   databaseokz+Grafana healtcheck failed. Database not ok.)re   r5   r9   r*   zGrafana running)re   r5   rI   grafana_org_idsession_namedashboard_uidsdashboard_datasourcez4Error fetching grafana endpoint. Is grafana running?exc_info)re   r5   	exception)rI   GRAFANA_HOST_DISABLED_VALUEdashboard_optional_utilsrest_responsedashboard_utilsHTTPStatusCodeOKrD   rE   rF   GRAFANA_IFRAME_HOST_ENV_VARGRAFANA_HEALTHCHECK_PATHhttp_sessionr9   INTERNAL_ERRORr*   r^   rl   r_   r[   	Exceptionloggerdebugr   )r8   reqgrafana_iframe_hostrM   respr*   r)   r#   r#   r$   grafana_health~   sd   

4zMetricsHead.grafana_healthz/api/prometheus_healthc              
      s   zS| j  dt }| jj|| jd4 I d H 0}|jdkr3tjtj	j
d|jdW  d   I d H  W S tjtj	jddW  d   I d H  W S 1 I d H sMw   Y  W d S  tyy } ztjd|d	 tjtj	j
dt|d
W  Y d }~S d }~ww )Nrf   headersrg   zprometheus healthcheck failed.rh   zprometheus running)re   r5   z:Error fetching prometheus endpoint. Is prometheus running?ro   )re   r5   reason)rL   PROMETHEUS_HEALTHCHECK_PATHrz   rF   r1   r9   rs   rt   ru   rv   r{   rw   r|   r}   r~   r   )r8   r   rM   r   r)   r#   r#   r$   prometheus_health   s<   

4zMetricsHead.prometheus_healthc           
   
   C   s  t j| jrt| j t j| jdd t j| jd}t j| jd}t	t j| jdd}|
tj|d W d   n1 sDw   Y  t j|d}t j|dd t	t j|d	d}|
tj| jd
 W d   n1 svw   Y  t jtt}tt jtt}g }t|tr|}nt|trt| }t j|d}t j|dd t j| jdd t	t j|d	d"}|
t|| jdd t|D dd t|D d W d   n1 sw   Y  t	t j| jdd}t \}	| jd< |
|	 W d   n	1 sw   Y  t	t j| jdd}t \}	| jd< |
|	 W d   n	1 s7w   Y  t	t j| jdd}t  \}	| jd< |
|	 W d   n	1 s`w   Y  t	t j| jdd}t! \}	| jd< |
|	 W d   n	1 sw   Y  t	t j| jdd}t" \}	| jd< |
|	 W d   n	1 sw   Y  t	t j| jdd}t# \}	| jd< |
|	 W d   dS 1 sw   Y  dS )zY
        Creates the Grafana configurations that are by default provided by Ray.
        Texist_okprovisioningzgrafana.iniw)grafana_provisioning_folderNrC   zdefault.yml)dashboard_output_folderdatasourcesc                 S   s$   i | ]\}\}}d |d  |qS )httpHeaderName   r#   )r    iheader_r#   r#   r$   
<dictcomp>)      
z?MetricsHead._create_default_grafana_configs.<locals>.<dictcomp>c                 S   s$   i | ]\}\}}d |d  |qS )httpHeaderValuer   r#   )r    r   r   valuer#   r#   r$   r   -  r   )rL   prometheus_namejsonDatasecureJsonDatazdefault_grafana_dashboard.jsondefaultzserve_grafana_dashboard.jsonservez'serve_deployment_grafana_dashboard.jsonserve_deploymentz serve_llm_grafana_dashboard.json	serve_llmzdata_grafana_dashboard.jsondataztrain_grafana_dashboard.jsontrain)$rD   rM   existsrU   shutilrmtreemakedirsrN   rV   openwriter   formatr   rX   rE   rF   rJ   rK   r3   r0   rP   r   r.   r,   r-   r   r[   	enumerater   r_   r   r   r	   r   r
   )
r8   r   'grafana_prov_folder_with_latest_sessionfdashboard_provisioning_pathrL   r1   prometheus_header_pairsdata_sources_pathcontentr#   r#   r$   _create_default_grafana_configs   sL  

	$z+MetricsHead._create_default_grafana_configsc                 C   s   t j| jdd}t j|rt | t jt j|dd t j| jt	}t
|d}|tj|d W d   dS 1 sAw   Y  dS )z\
        Creates the Prometheus configurations that are by default provided by Ray.
        
prometheuszprometheus.ymlTr   r   )(prom_metrics_service_discovery_file_pathN)rD   rM   rN   rS   r   remover   dirnamerQ   r   r   r   r   r   )r8   prometheus_config_output_pathprom_discovery_file_pathr   r#   r#   r$   "_create_default_prometheus_configsy  s    
"z.MetricsHead._create_default_prometheus_configsc                    s&   t   I d H  |   |   d S r   )r6   runr   r   )r8   r:   r#   r$   r     s   zMetricsHead.runc              	      s   | j j| j dt| | jd4 I d H &}|jdkr/| I d H }|W  d   I d H  S | I d H }t|j|1 I d H sBw   Y  d S )Nz/api/v1/query?query=r   rg   )	rz   rF   rL   r   r1   r9   r*   textr4   )r8   queryr   	prom_datar5   r#   r#   r$   _query_prometheus  s   
zMetricsHead._query_prometheus)r<   r=   r>   r7   routesrF   aiohttpwebResponser   r   r   r   r   r   r?   r#   r#   r:   r$   r@   T   s    )7
 'r@   );r*   loggingrD   r   urllib.parser   r   ray.dashboard.optional_utils	dashboardoptional_utilsrs   ray.dashboard.utilsutilsru   ray._private.ray_constantsr   r   7ray.dashboard.modules.metrics.grafana_dashboard_factoryr   r   r   r   r	   r
   'ray.dashboard.modules.metrics.templatesr   r   r   r   !ray.dashboard.subprocesses.moduler   !ray.dashboard.subprocesses.routesr   r   	getLoggerr<   r}   setLevelINFOrR   rK   rJ   rP   r0   rZ   rY   r   rH   rG   r\   r]   rr   rx   rW   ry   r3   r|   r4   r@   r#   r#   r#   r$   <module>   sD     
	