o $i@sddlZddlmZddlmZmZmZmZddlm Z ddl Z ddl m Z ddlmZGddde ZGd d d eZed d Gd ddeZdS)N)abstractmethod)DictListOptionalUnion)Callback)TensorflowCheckpoint) PublicAPIcseZdZdZgdZd(deeeefffdd Zde defd d Z d)d d Z d)ddZ d)ddZ d)ddZd)ddZd)ddZd)ddZd)ddZd)ddZd)ddZd)d d!Zd)d"d#Zd)d$d%Zd)d&d'ZZS)* _Callbackz+Base class for Ray Train's Keras callbacks.) epoch_begin epoch_endtrain_batch_begintrain_batch_endtest_batch_begintest_batch_endpredict_batch_beginpredict_batch_end train_begin train_end test_begintest_end predict_begin predict_endvalidation_endoncsPttt|ts|g}tfdd|Dr#td|j|_ dS)Nc3s|]}|jvVqdSN)_allowed).0wselfW/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/ray/train/tensorflow/keras.py %sz%_Callback.__init__..z4Invalid trigger time selected: {}. Must be one of {}) superr __init__ isinstancelistany ValueErrorformatr_on)r r __class__rr"r% s  z_Callback.__init__logswhencCtrNotImplementedError)r r.r/r!r!r"_handle-sz_Callback._handleNcCd|jvr ||ddSdS)Nr r+r3r epochr.r!r!r"on_epoch_begin0 z_Callback.on_epoch_begincCr4)Nr r5r6r!r!r" on_epoch_end4r9z_Callback.on_epoch_endcCr4)Nr r5r batchr.r!r!r"on_train_batch_begin8r9z_Callback.on_train_batch_begincCr4)Nrr5r;r!r!r"on_train_batch_end<r9z_Callback.on_train_batch_endcCr4)Nrr5r;r!r!r"on_test_batch_begin@r9z_Callback.on_test_batch_begincCr4)Nrr5r;r!r!r"on_test_batch_endDr9z_Callback.on_test_batch_endcCr4)Nrr5r;r!r!r"on_predict_batch_beginHr9z _Callback.on_predict_batch_begincCr4)Nrr5r;r!r!r"on_predict_batch_endLr9z_Callback.on_predict_batch_endcCd|jvr ||ddSdS)Nrr5r r.r!r!r"on_train_beginPr9z_Callback.on_train_begincCrC)Nrr5rDr!r!r" on_train_endTr9z_Callback.on_train_endcCrC)Nrr5rDr!r!r" on_test_beginXr9z_Callback.on_test_begincCrC)Nrr5rDr!r!r" on_test_end\r9z_Callback.on_test_endcCrC)Nrr5rDr!r!r"on_predict_begin`r9z_Callback.on_predict_begincCrC)Nrr5rDr!r!r"on_predict_enddr9z_Callback.on_predict_end)rr)__name__ __module__ __qualname____doc__rrstrrr%rr3r8r:r=r>r?r@rArBrErFrGrHrIrJ __classcell__r!r!r,r"r s&              r c seZdZ   ddeeeefdeeeefdeeeeeeeeffffdd Zded efd d Z e ded e fd dZ e defddZ dedefddZZS)RayReportCallbackr N checkpoint_onreport_metrics_onmetricscsTt|tr|g}t|tr|g}tt||}tj|d||_||_||_dS)N)r) r&rOr'setr$r%_checkpoint_on_report_metrics_on_metrics)r rRrSrTrr,r!r"r%js   zRayReportCallback.__init__r.returncst|jtdtttfsJ|jdur}n2t|jtr%|j|ji}n#t|jtr6fdd|jD}nt|jtrHfdd|jD}t|tsOJ|S)Ncsi|]}||qSr!r!)rmetricr.r!r" sz;RayReportCallback._get_reported_metrics..csi|] \}}||qSr!r!)rkeyrZr[r!r"r\s)r&rXtyperOr'dictitems)r r.reported_metricsr!r[r"_get_reported_metrics|s     z'RayReportCallback._get_reported_metrics checkpointcCr0)CSave checkpoint and report metrics corresonding to this checkpoint.r1r rTrcr!r!r"_save_and_report_checkpointsz-RayReportCallback._save_and_report_checkpointcCr0)Report metrics.r1r rTr!r!r"_report_metricssz!RayReportCallback._report_metricsr/cCsj||jvs ||jvs J||}||jv}|r.t|j}|||tj|j dddS| |dS)NT) ignore_errors) rVrWrbr from_modelmodelrfshutilrmtreepathri)r r.r/rTshould_checkpointrcr!r!r"r3s    zRayReportCallback._handle)r r N)rKrLrMrrOrrrr%rbrrrfrir3rPr!r!r,r"rQis* rQalpha) stabilityc@s0eZdZdZdedefddZdefddZdS) ReportCheckpointCallbackaKeras callback for Ray Train reporting and checkpointing. .. note:: Metrics are always reported with checkpoints, even if the event isn't specified in ``report_metrics_on``. Example: .. testcode:: python ############# Using it in TrainSession ############### from ray.air.integrations.keras import ReportCheckpointCallback def train_loop_per_worker(): strategy = tf.distribute.MultiWorkerMirroredStrategy() with strategy.scope(): model = build_model() model.fit(dataset_shard, callbacks=[ReportCheckpointCallback()]) Args: metrics: Metrics to report. If this is a list, each item describes the metric key reported to Keras, and it's reported under the same name. If this is a dict, each key is the name reported and the respective value is the metric key reported to Keras. If this is None, all Keras logs are reported. report_metrics_on: When to report metrics. Must be one of the Keras event hooks (less the ``on_``), e.g. "train_start" or "predict_end". Defaults to "epoch_end". checkpoint_on: When to save checkpoints. Must be one of the Keras event hooks (less the ``on_``), e.g. "train_start" or "predict_end". Defaults to "epoch_end". rTrccCstjj||ddS)rdrcNraytrainreportrer!r!r"rfsz4ReportCheckpointCallback._save_and_report_checkpointcCstjj|dddS)rgNrtrurhr!r!r"risz(ReportCheckpointCallback._report_metricsN)rKrLrMrNrrrfrir!r!r!r"rss  rs)rmabcrtypingrrrrtensorflow.keras.callbacksr KerasCallbackrvray.train.tensorflowrray.util.annotationsr r rQrsr!r!r!r"s    ]?