o
    ciF                     @   s   d dl mZ d dlmZmZ d dlZd dlmZmZm	Z	m
Z
 d dlmZmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ G dd dee
ZG dd de	eZdS )    )contextmanager)DictOptionalN)ControllerCallbackTrainContextCallbackWorkerCallbackWorkerGroupCallback)TrainRunContextget_train_context)TrainControllerStateTrainControllerStateType)Metric)ControllerMetrics)WorkerMetrics)time_monotonicc                   @   sP   e Zd ZdZdefddZdd Zdedefd	d
Ze	dd Z
e	dd ZdS )ControllerMetricsCallbackz2Callback that records controller-specific metrics.train_run_contextc                 C   s>   |  j| _|j| _t| j| j| _| jtj 	t
j dS )z+Initialize metrics after controller starts.N)get_run_configname	_run_namerun_id_run_idr   get_controller_metrics_metricsCONTROLLER_STATErecordr   INITIALIZINGselfr    r   \/home/ubuntu/.local/lib/python3.10/site-packages/ray/train/v2/_internal/callbacks/metrics.pyafter_controller_start   s   z0ControllerMetricsCallback.after_controller_startc                 C      | j  D ]}|  qdS )z.Shutdown metrics before controller shuts down.Nr   valuesresetr   metricr   r   r    before_controller_shutdown%      
z4ControllerMetricsCallback.before_controller_shutdownprevious_statecurrent_statec                 C   s   | j tj |j dS )z8Record state transitions after controller state updates.N)r   r   r   r   _state_type)r   r*   r+   r   r   r    after_controller_state_update*   s   z7ControllerMetricsCallback.after_controller_state_updatec                 c   .    t  }dV  t  | }| jtj | dS )z)Measure time taken to start worker group.N)r   r   r   WORKER_GROUP_START_TOTAL_TIME_Sr   r   start_time_selapsed_time_sr   r   r    on_worker_group_start4      
z/ControllerMetricsCallback.on_worker_group_startc                 c   r.   )z,Measure time taken to shutdown worker group.N)r   r   r   "WORKER_GROUP_SHUTDOWN_TOTAL_TIME_Sr   r0   r   r   r    on_worker_group_shutdown>   r4   z2ControllerMetricsCallback.on_worker_group_shutdownN)__name__
__module____qualname____doc__r	   r!   r(   r   r-   r   r3   r6   r   r   r   r    r      s    


	r   c                   @   s:   e Zd ZdZdefddZdd Zdd Zed	d
 Z	dS )WorkerMetricsCallbackz.Callback that records worker-specific metrics.r   c                 C   s   |  j| _|j| _d | _d S )N)r   r   r   r   r   r   r   r   r   r    __init__L   s   
zWorkerMetricsCallback.__init__c                 C   s:   t  }tj }| }| }t| j| j	||| _
dS )z6Initialize metrics after train context is initialized.N)r
   rayruntime_contextget_runtime_contextget_world_rankget_actor_idr   get_worker_metricsr   r   r   )r   train_contextcore_context
world_rankworker_actor_idr   r   r    after_init_train_contextQ   s   

z.WorkerMetricsCallback.after_init_train_contextc                 C   r"   )z!Shutdown metrics before shutdown.Nr#   r&   r   r   r    before_shutdown[   r)   z%WorkerMetricsCallback.before_shutdownc                 c   r.   )zb
        Context manager to measure the time taken to report a checkpoint to the storage.
        N)r   r   r   REPORT_TOTAL_BLOCKED_TIME_Sr   r0   r   r   r    	on_report`   s
   
zWorkerMetricsCallback.on_reportN)
r7   r8   r9   r:   r	   r<   rG   rH   r   rJ   r   r   r   r    r;   I   s    
r;   )
contextlibr   typingr   r   r=   )ray.train.v2._internal.execution.callbackr   r   r   r   (ray.train.v2._internal.execution.contextr	   r
   1ray.train.v2._internal.execution.controller.stater   r   #ray.train.v2._internal.metrics.baser   )ray.train.v2._internal.metrics.controllerr   %ray.train.v2._internal.metrics.workerr   ray.train.v2._internal.utilr   r   r;   r   r   r   r    <module>   s    3