o miA@sbdZddlZddlZddlZddlZddlZddlmZddlZ ddl Z ddl m mZddlZddlmZddlmZmZddlmZddZd efd d Zd ejvrYened eeeZd dZddZ ddZ!ddZ"ddZ#ddZ$ddZ%ddZ&e 'Z(e!ddZ)e)Z*Gdd d e*Z+Gd!d"d"e j j,j-Z.Gd#d$d$e j j,j-Z/dS)%z keras init.N)chain)ValidationDataLogger) Deprecated deprecate)add_import_hookcCsBddlm}ddlm}|||dkrtd|ddSdS)Nr) __version__ parse_versionz2.4.0zKeras version z0 is not fully supported. Required keras >= 2.4.0)kerasr wandb.utilr wandbtermwarn) keras_versionr r[/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/wandb/integration/keras/keras.py_check_keras_versions   rreturncCs&ddlm}|tj|dkrdSdS)zFFLOPS computation is restricted to TF 2.x as it requires tf.compat.v1.rrz2.0.0TF)r r tfrrrrr_can_compute_flopss rr cCsFtjd}|r!t|dr!|jf}t|dr||jf}t||SdS)Nz&tensorflow.python.data.ops.dataset_ops DatasetV2 DatasetV1F)r util get_modulehasattrrr isinstance)data dataset_ops dataset_typesrrr is_dataset2s    rcCsttjjjf}tjd}|r+||jf}t|dr ||j f}n t|dr+||j f}t|dp9t|dp9t ||S)Nz'tensorflow.python.data.ops.iterator_ops EagerIterator IteratorV2next__next__) rr utilsSequencer rrIteratorrrr r)rtypes iterator_opsrrris_generator_like=s      r(c s<ddlmddlm}|d|tjkr|dkrLnn-d}zddlm}ddlm}dd lm }WnZt t fyKt d td YdSwd }ddlm}zddlm}dd lm }Wn-t t fyz dd lm}m}Wnt t fyt d td YYdSwYnwt jd}t j|d}|r|jjn|r|jj|j|jfddfdd}fdd}fdd} |_||_|_||_t jd|ddgt jd|ddg|r| |j_t jdddgdS|r| |j_t jd|ddgdSdS) Nrcontextrz2.6.0z2.13.0z keras.engine)training)training_arrays_v1)training_generator_v1z Unable to patch Tensorflow/Kerasz(exception while trying to patch_tf_kerasztensorflow.python.keras.engine)training_arraystraining_generatorz*tensorflow.python.keras.engine.training_v2z .training_v1cst|trBtr|_dStr$rt|_dStddStt r=tdt j r=fdd}||_dS|_ dSdS)Nz.set_wandb_attrs..gen) r WandbCallbackr( generatorrexecuting_eagerlyiterr r tuplerTensorvalidation_data)cbkr5r6r)r4rset_wandb_attrss     z'patch_tf_keras..set_wandb_attrscsV|dg}|d}|d}|r$|r$|D] }||d|dfq|i|S)N callbacks val_inputs val_targetsrget)argskwargscbksrArBr>) old_arraysr?rr new_arrayss   z"patch_tf_keras..new_arraysc<|dg}|d}|r|D]}||q|i|SNr@r=rCrErFrGr5r>) old_generatorr?rr new_generator   z%patch_tf_keras..new_generatorcrJrKrCrL)old_v2r?rrnew_v2rOzpatch_tf_keras..new_v2r z.training_arraysfit_loopz.training_generator fit_generatorz/tensorflow.python.keras.engine.training_v2.Loopfitz.training.Model)tensorflow.python.eagerr*r r rr keras.enginer+r,r- ImportErrorAttributeErrorr termerrorlogger exceptiontensorflow.python.keras.enginer.r/rrLooprTModelrRrS orig_fit_looporig_fit_generatorpatchedappend) r keras_enginer+r.r/ training_v2_1 training_v2_2rIrNrQr)r*rHrMrPr?rpatch_tf_kerasLsz                  rfcCs t|dS)Ndtype)rarrayrrr_array_has_dtype rjcCs:t|s t|dSt|st|dSt|||<dSr0)rj_warn_not_loggingis_numeric_array_warn_not_logging_non_numericr Histogram)metricskeyvaluesrrr_update_if_numericsrscCst|jtjSr0)np issubdtypergnumberrhrrrrmrmcCtjd|ddddS)Nz#Non-numeric values found in layer: z, not logging this layerFrepeatr r namerrrrn  rncCrx)NzLayer z1 has undetermined datatype not logging this layerFryr{r|rrrrlr~rlcCs<ddlm}|tj|dkrtjjjj}|Stjjj}|S)Nrrz2.9.0)r r rrr optimizerslegacy Optimizer)r custom_optimizer_parent_classrrr"_get_custom_optimizer_parent_classs   rcs8eZdZfddZddZddZfddZZS) _CustomOptimizercs.tjddt|j|_t|j|_dS)NCustomOptimizerr|)super__init__rfunction_resource_apply_dense_resource_apply_sparseself __class__rrrsz_CustomOptimizer.__init__cCs||dSr0)assign)rgradvarrrrrsz&_CustomOptimizer._resource_apply_densecCdSr0r)rrrindicesrrrrz'_CustomOptimizer._resource_apply_sparsecs tSr0)r get_configrrrrr rkz_CustomOptimizer.get_config)__name__ __module__ __qualname__rrrr __classcell__rrrrrs  rcs2eZdZdZfddZd ddZddZZS) _GradAccumulatorCallbackzbAccumulates gradients during a fit() call when used in conjunction with the CustomOptimizer above.cs,t|||_dd|jD|_dS)NcSsg|] }tt|jqSr)rtzerosr;shape).0wrrr sz6_GradAccumulatorCallback.set_model..)r set_model get_weights og_weightstrainable_weightsgradsrmodelrrrrs  z"_GradAccumulatorCallback.set_modelNcCs8t|j|jjD] \}}||7}q|j|jdSr0)ziprrrnumpy set_weightsr)rbatchlogsgrrrr on_batch_endsz%_GradAccumulatorCallback.on_batch_endcCsdd|jDS)NcSsg|]}|qSr)copy)rrrrrrz6_GradAccumulatorCallback.get_grads..)rrrrr get_gradsrwz"_GradAccumulatorCallback.get_gradsr0)rrr__doc__rrrrrrrrrs   rcsveZdZdZ                         dKd d Zd d ZddZddZddZddZ fddZ dLddZ dMddZ dMddZ dMddZdMd d!ZdMd"d#ZdMd$d%ZdMd&d'ZdMd(d)ZdMd*d+ZdMd,d-ZdMd.d/ZdMd0d1ZdMd2d3ZdMd4d5ZdMd6d7Zd8d9Zd:d;ZdNdd?Zd@dAZdBdCZ dDdEZ!dFdGZ"dHe#fdIdJZ$Z%S)Or7a`WandbCallback` automatically integrates keras with wandb. Example: ```python model.fit( X_train, y_train, validation_data=(X_test, y_test), callbacks=[WandbCallback()], ) ``` `WandbCallback` will automatically log history data from any metrics collected by keras: loss and anything passed into `keras_model.compile()`. `WandbCallback` will set summary metrics for the run associated with the "best" training step, where "best" is defined by the `monitor` and `mode` attributes. This defaults to the epoch with the minimum `val_loss`. `WandbCallback` will by default save the model associated with the best `epoch`. `WandbCallback` can optionally log gradient and parameter histograms. `WandbCallback` can optionally save training and validation data for wandb to visualize. Args: monitor: (str) name of metric to monitor. Defaults to `val_loss`. mode: (str) one of {`auto`, `min`, `max`}. `min` - save model when monitor is minimized `max` - save model when monitor is maximized `auto` - try to guess when to save the model (default). save_model: True - save a model when monitor beats all previous epochs False - don't save models save_graph: (boolean) if True save model graph to wandb (default to True). save_weights_only: (boolean) if True, then only the model's weights will be saved (`model.save_weights(filepath)`), else the full model is saved (`model.save(filepath)`). log_weights: (boolean) if True save histograms of the model's layer's weights. log_gradients: (boolean) if True log histograms of the training gradients training_data: (tuple) Same format `(X,y)` as passed to `model.fit`. This is needed for calculating gradients - this is mandatory if `log_gradients` is `True`. validation_data: (tuple) Same format `(X,y)` as passed to `model.fit`. A set of data for wandb to visualize. If this is set, every epoch, wandb will make a small number of predictions and save the results for later visualization. In case you are working with image data, please also set `input_type` and `output_type` in order to log correctly. generator: (generator) a generator that returns validation data for wandb to visualize. This generator should return tuples `(X,y)`. Either `validate_data` or generator should be set for wandb to visualize specific data examples. In case you are working with image data, please also set `input_type` and `output_type` in order to log correctly. validation_steps: (int) if `validation_data` is a generator, how many steps to run the generator for the full validation set. labels: (list) If you are visualizing your data with wandb this list of labels will convert numeric output to understandable string if you are building a multiclass classifier. If you are making a binary classifier you can pass in a list of two labels ["label for false", "label for true"]. If `validate_data` and generator are both false, this won't do anything. predictions: (int) the number of predictions to make for visualization each epoch, max is 100. input_type: (string) type of the model input to help visualization. can be one of: (`image`, `images`, `segmentation_mask`, `auto`). output_type: (string) type of the model output to help visualization. can be one of: (`image`, `images`, `segmentation_mask`, `label`). log_evaluation: (boolean) if True, save a Table containing validation data and the model's predictions at each epoch. See `validation_indexes`, `validation_row_processor`, and `output_row_processor` for additional details. class_colors: ([float, float, float]) if the input or output is a segmentation mask, an array containing an rgb tuple (range 0-1) for each class. log_batch_frequency: (integer) if None, callback will log every epoch. If set to integer, callback will log training metrics every `log_batch_frequency` batches. log_best_prefix: (string) if None, no extra summary metrics will be saved. If set to a string, the monitored metric and epoch will be prepended with this value and stored as summary metrics. validation_indexes: ([wandb.data_types._TableLinkMixin]) an ordered list of index keys to associate with each validation example. If log_evaluation is True and `validation_indexes` is provided, then a Table of validation data will not be created and instead each prediction will be associated with the row represented by the `TableLinkMixin`. The most common way to obtain such keys are is use `Table.get_index()` which will return a list of row keys. validation_row_processor: (Callable) a function to apply to the validation data, commonly used to visualize the data. The function will receive an `ndx` (int) and a `row` (dict). If your model has a single input, then `row["input"]` will be the input data for the row. Else, it will be keyed based on the name of the input slot. If your fit function takes a single target, then `row["target"]` will be the target data for the row. Else, it will be keyed based on the name of the output slots. For example, if your input data is a single ndarray, but you wish to visualize the data as an Image, then you can provide `lambda ndx, row: {"img": wandb.Image(row["input"])}` as the processor. Ignored if log_evaluation is False or `validation_indexes` are present. output_row_processor: (Callable) same as `validation_row_processor`, but applied to the model's output. `row["output"]` will contain the results of the model output. infer_missing_processors: (bool) Determines if `validation_row_processor` and `output_row_processor` should be inferred if missing. Defaults to True. If `labels` are provided, we will attempt to infer classification-type processors where appropriate. log_evaluation_frequency: (int) Determines the frequency which evaluation results will be logged. Default 0 (only at the end of training). Set to 1 to log every epoch, 2 to log every other epoch, and so on. Has no effect when log_evaluation is False. compute_flops: (bool) Compute the FLOPs of your Keras Sequential or Functional model in GigaFLOPs unit. val_lossrautoFTN$best_cKstjdur tdttjddtjjjtjd }d|j _ Wdn1s)wYd|_ | dur?t | r<| } n| |_ | durEg} | |_ t| d|_||_||_||_||_tdtjtjjd|_||_|ruttjddd|_||_||_||_| |_ d |_!|"d d}|durttj#d d|} | |_$||_%||_&||_'|durt()|nd|_*||_+||_,||_-d|_.|jrt/t0j12d d dkrt3d|jdurt4dt5|jt6t7frt8|jdkrt4d|j\|_9|_:n|j|_9d|_:|dvr t;d|dd}|dkrtt?d|_@n/|dkr)tt?d|_@n d|jvs6|jBdr@tt?d|_@n tt?d|_@tjjC"|j,|j}|dur^||_@d|_D||_E||_F||_G||_H||_Id |_JdS)Nz1You must call wandb.init() before WandbCallback()zWandbCallback is deprecated and will be removed in a future release. Please use the WandbMetricsLogger, WandbModelCheckpoint, and WandbEvalCallback callbacks instead. See https://docs.wandb.ai/guides/integrations/keras for more information.) field_namewarning_message)r3Tdz model-best.h5a<The save_model argument by default saves the model in the HDF5 format that cannot save custom objects like subclassed models and custom layers. This behavior will be deprecated in a future release in favor of the SavedModel format. Meanwhile, the HDF5 model is saved as W&B files and the SavedModel as W&B Artifacts.F data_typezThe data_type argument of wandb.keras.WandbCallback is deprecated and will be removed in a future release. Please use input_type instead. Setting input_type = data_type..rz3Gradient logging requires tensorflow 2.0 or higher.z8training_data argument is required for gradient logging.z+training data must be a tuple of length two)rminmaxzWandbCallback mode z# is unknown, fallback to auto mode.rrinfrz-infaccfmeasure)Kr r3Errorrrkeras_callback wandb_lib telemetryr*featurer r=r(labelsr predictionsmonitorverbosesave_weights_only save_graphsaveospathjoindirfilepath save_modelkeras_callback__save_modelsave_model_as_artifact log_weights log_gradients training_datar8_graph_renderedrDkeras_callback__data_type input_type output_typelog_evaluationvalidation_stepsrtri class_colorslog_batch_frequencylog_best_prefix compute_flops_prediction_batch_sizeintrrsplit Exception ValueErrorrlistr;len_training_data_x_training_data_yprintoperatorlt monitor_opfloatbestgt startswithsummary_validation_data_logger_validation_indexes_validation_row_processor_prediction_row_processor_infer_missing_processors_log_evaluation_frequency_model_trained_since_last_eval)rrrmoderrrrrr=rrr8rrrrrrrrvalidation_indexesvalidation_row_processorprediction_row_processorinfer_missing_processorslog_evaluation_frequencyrrFtelr previous_bestrrrrs                 zWandbCallback.__init__cCsN|jj}||}tjj||}|j|jjtdd|_ ||_ t |_ dS)N)loss optimizerT) rinputsrr modelsr^compilerr_wandb_internal_model_grad_accumulator_modelr_grad_accumulator_callback)rroutputsgrad_acc_modelrrr_build_grad_accumulator_models  z+WandbCallback._build_grad_accumulator_modelcC |jduSr0rrrrr_implements_train_batch_hooks)rkz+WandbCallback._implements_train_batch_hookscCrr0rrrrr_implements_test_batch_hooks,rkz*WandbCallback._implements_test_batch_hookscCrr0rrrrr_implements_predict_batch_hooks/rkz-WandbCallback._implements_predict_batch_hookscCs ||_dSr0)params)rrrrr set_params2rkzWandbCallback.set_paramscst||jdkrt|jdkrtjj|jdjdd|_|jr9|j dur9t|j dkr9tj|j dj|_ |j rB| dSdS)NrrT)risky) rrrrrr rguess_data_typerrr rr rrrrr5s   zWandbCallback.set_modelc Cs|jrB|jrDz|jstdWdS|jj|j|jj|dd|_WdSt yA}ztdt |WYd}~dSd}~wwdSdS)Nz/WandbCallback unable to read model from trainer)rcommitFz+Error during prediction logging for epoch: ) rrrr r log_predictionsmake_predictionspredictrrstr)rrerrr_attempt_evaluation_log@s   z%WandbCallback._attempt_evaluation_logc Cs|duri}|jrtj|dd|jrtj|dd|jdvs(|jdvrT|jr1t |j|_ |j durz WandbCallback.on_train_batch_endcCrr0rrrrrr on_test_beginrzWandbCallback.on_test_begincCrr0rrArrr on_test_endrzWandbCallback.on_test_endcCrr0rr6rrron_test_batch_beginrz!WandbCallback.on_test_batch_begincCrr0rr6rrron_test_batch_endrzWandbCallback.on_test_batch_endc Csp|jrzfd}|jr |j}nD|jrL|jstdn8d}d}t|jD]$}t|j\}}|dur5||}}q"tj ||ddtj ||dd}}q"||f}ntd|rht |d|d|j |j |j |j|jd|_Wnty}ztdt|WYd}~nd}~ww|jrtrz |tjd<WdSty}ztd t|WYd}~dSd}~wwdSdS) NzzWandbCallback is unable to log validation data. When using a generator for validation_data, you must pass validation_stepsraxiszWandbCallback is unable to read validation_data from trainer and therefore cannot log validation data. Ensure Keras is properly patched by calling `from wandb.keras import WandbCallback` at the top of your script.r)rtargetsindexesrr class_labelsrzcError initializing ValidationDataLogger in WandbCallback. Skipping logging validation data. Error: GFLOPsz'Unable to compute FLOPs for this model.)rr=r8rr r ranger!rtrbrrrrrrrrrrr get_flopsrrZr[) rrr=xy_true_bxby_truerrrron_train_beginsf     zWandbCallback.on_train_begincCs|jr |dSdSr0)rrrArrr on_train_ends zWandbCallback.on_train_endcCrr0rrArrron_predict_beginrzWandbCallback.on_predict_begincCrr0rrArrron_predict_endrzWandbCallback.on_predict_endcCrr0rr6rrron_predict_batch_beginrz$WandbCallback.on_predict_batch_begincCrr0rr6rrron_predict_batch_endrz"WandbCallback.on_predict_batch_endc sdjddkr1tjdkrfddD}|Stjdkr(tdddD}|Stjtdd }tjdkrcg}|D]}z |j|WqFt y`||YqFw|S|}|S) Nrrrcs,g|]}ddkrjdnjdqS)rg?r)rrlogitlogitsrrrrsz5WandbCallback._logits_to_captions..zpkeras model is producing a single output, so labels should be a length two array: ["False label", "True label"].cSsg|]}|dqS)rrrZrrrrrrF) rrrr r rtargmaxstackrb IndexError)rr]captionsrlabelrr\r_logits_to_captionss0  z!WandbCallback._logits_to_captionscCsjt|djdks|djddkr|S|jdur|jn ttj|djd}|tj|dd}|S)NrrrYrrF)rrrrtrir rr^)rmasksrimgsrrr_masks_to_pixelss$ zWandbCallback._masks_to_pixelscs|jd}|jd}t|}||krtjj||dd}nt|}g}g}|D]}||} || |||q&|jjrN|jj t |dd} |j n#|jj t ||j d} t| t|krqd|_ |jj t ||j d} |j dkr|jdvr|||jdkr|| n| } |jdkr||n|} fd d t| D} fd d t| D}ttt| |SdS|j dvr,|j dkr||n|}|jdkr|| fd d t|DS|jdvr%|jdkr|| n| } |jdkr||n|} d d t|D}dd t| D} dd t| D}ttt|| |Sdd |DS|jdvrh|jdkr=|| n| } |jdkrJ||n|} dd t| D} dd t| D}ttt| |SdS)NrrF)replace batch_sizerbr r#cs$g|]\}}tj||ddqS)r)captiongroupingr ImagerirrarrrMsz-WandbCallback._log_images..c"g|] \}}tj||dqS)rjrlrnrprrrQcrqrrrlrnrprrr_rscSg|] \}}tj|ddqS)rkrlrnrrrrn cSg|] \}}t|qSrrlrnrrrrrcSrxrrlrnrrrrurycSsg|]}t|qSrrl)rimgrrrrscSrt)rrvrlrnrrrrrwcSrxrrlrnrrrrry)r=rrtrandomchoicerLrbrstatefulrr_ reset_statesrrrrcrf enumeraterr from_iterabler)rr% validation_X validation_yvalidation_lengthr test_data test_outputro test_exampleroutput_image_datareference_image_data output_imagesreference_imagesinput_image_data input_imagesrrprr0 s                                zWandbCallback._log_imagescCsi}|jjD]:}|}t|dkr t|d|jd|dqt|dkr@t|d|jd|dt|d|jd|dq|S)Nrz parameters/z.weightsrrz.bias)rlayersrrrsr})rrplayerweightsrrrr.s    zWandbCallback._log_weightscCstj}td|jj|j|jd|jgdt||jj }|jj }i}t ||D]\}}t ||d|jddd<q*|S)NERRORr)rr@z gradients/:z .gradient) tf_loggerlevelsetLevelr rTrrr rrrrr ror}r)rog_levelrrrpweightrrrrr/s   zWandbCallback._log_gradientscCs8d\}}}|jr|jd|jd}}|j|}nI|jrc|js'tddSt|jD]6}t|j\}}|j|}|durH|||}}}q,t j ||ddt j ||ddt j ||dd}}}q,|j dvrw|j dkrwtj ||||jdS|j dvr|j d krtj||||j|jd Std |j d |j dS) N)NNNrrzdwhen using a generator for validation data with dataframes, you must pass validation_steps. skippingrF)r!r"rb)rNrOy_predrr#)rNrOrrrz&unknown dataframe type for input_type=z and output_type=)r=rrr8rr r rLr!rtrbrrimage_categorizer_dataframerimage_segmentation_dataframer)rrNrOrrPrQrRby_predrrr_log_dataframesH      zWandbCallback._log_dataframec CstjjrdS|jdkr%td|dd|jd|jdd|jdd|j z|j r5|j j |jd d WdS|j j |jd d WdSt tttfya}ztd t|WYd}~dSd}~ww) Nrr'r(r)r*r+r,z, saving model to T) overwritezfCan't save model in the h5py format. The model will be saved as as an W&B Artifact in the 'tf' format.)r r3disabledrrrrr1rrr save_weightsrrW RuntimeError TypeErrorrXrYrZr[)rr&rrrrr3s& &zWandbCallback._save_modelcCstjjrdS|jj|jdddddtjdtjj}tj |dd}| |jddtjj |dd |gd t |jdddS) NTr)r save_formatzmodel-r)typelatestepoch_)aliases)r r3rrrrrmake_artifact_name_safer}Artifactadd_dir log_artifactshutilrmtree)rr&r}model_artifactrrrr4sz%WandbCallback._save_model_as_artifactrc st|ds tdt|jtjjjtjjj fst dddl m }dfdd|jj D}t|j|}||\}}tjj}tjjjtjjj}tjjjj|j|d |d }|jd d S) zCalculate FLOPS [GFLOPs] for a tf.keras.Model or tf.keras.Sequential model in inference mode. It uses tf.compat.v1.profiler under the hood. rz0self.model must be set before using this method.z]Calculating FLOPS is only supported for `tf.keras.Model` and `tf.keras.Sequential` instances.r)*convert_variables_to_constants_v2_as_graphrcs*g|]}tg|jdd|jqS)rN)r TensorSpecrrg)rinprhrrr)sz+WandbCallback.get_flops..scope)r:run_metacmdoptionsgeAr)rr rrrrr r Sequentialr^r0tensorflow.python.framework.convert_to_constantsrrrget_concrete_functioncompatv1 RunMetadataprofilerProfileOptionBuilderfloat_operationwith_empty_outputbuildprofiler:total_float_ops) rrr real_model frozen_funcrProptsflopsrrhrrMs4         zWandbCallback.get_flops)rrrFFFTNNNrNNNFNNNrTNNNTrF)Tr0)r)&rrrrrr rrrrrrr5r7rr?r@rBrCrDrErSrTrUrVrWrXrcrfr0r.r/rr3r4rrMrrrrrr7#szb     7       6    " u/r7)0rloggingrrrsys itertoolsrrrt tensorflowrtensorflow.keras.backendr backendr1r (wandb.sdk.integration_utils.data_loggingrwandb.sdk.lib.deprecaterrr rrboolrmodules getLoggerrrZrr(rfrjrsrmrnrl get_loggerrr_custom_optimizer_parent_classrr@Callbackrr7rrrrsF       y