o
    Ti7                  	   @   sB  d dl Z e je je jeddZe je je jeddZe je je jeddZe je je jeddZ	dZ
dZd	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"e Z#dZ$dZ%dZ&d Z'd!Z(dZ)d"Z*dZ+d#Z,d	Z-d$Z.d%Z/d&Z0d	Z1d'Z2dZ3d(Z4d	Z5d)Z6d*Z7d+Z8d,Z9d-Z:d,Z;d.Z<e8Z=d/Z>dZ?d0Z@d1ZAd2ZBdZCd3ZDdZEd4ZFdZGd5ZHdZIeBeCeDeEeFeGeHeIiZJd6d6d6d7ZKd	d8didd9ZLd:d8d iiZMd:d	g d;g d;d<iZNd:d=ddgddgg d;g d;ddgd>iZOd:dddgddgg d;ddgg d;ddgd?iZPd@ZQdAZRdS )B    Nconfig_templatesztemplate_zero0.jsonztemplate_zero1.jsonztemplate_zero2.jsonztemplate_zero3.jsong?	ds_config   z
autotuner should be enabled as:
"session_params": {
  "autotuning": {
    "enabled": true,
    "start_step": 5,
    "end_step": 15
    }
}

autotuningenabledFfastTresults_dirautotuning_resultsexps_dirautotuning_exps	overwritestart_profile_step   end_profile_step   metric_path
tuner_type
gridsearchrandommodel_basedtuner_early_stoppingtuner_num_trials2   arg_mappingsmax_train_batch_sizemin_train_batch_size"max_train_micro_batch_size_per_gpui   "min_train_micro_batch_size_per_gpunum_tuning_micro_batch_sizesmp_sizemetriclatency
throughputflopsforwardstepmodel_info_pathzX
"model_info": {
  "num_params": 1000000000,
  "hidden_size": 10,
  "num_layers": 12,
}

model_infoprofile
num_paramshidden_size
num_layersauto)train_batch_sizetrain_micro_batch_size_per_gpugradient_accumulation_stepsstage)r.   zero_optimizationmemory_break_downr1   )g    ׇAg    eͽAg    eA)r0   reduce_bucket_sizeallgather_bucket_size   )r0   overlap_commreduce_scatterr3   r4   contiguous_gradients)r0   r6   r7   r3   allgather_partitionsr4   r8   globalz)Sospathjoindirnamerealpath__file__DEFAULT_TEMPLATE_PATH_ZERO_0DEFAULT_TEMPLATE_PATH_ZERO_1DEFAULT_TEMPLATE_PATH_ZERO_2DEFAULT_TEMPLATE_PATH_ZERO_3METRIC_PERCENT_DIFF_CONST	DS_CONFIGBUFSIZEAUTOTUNING_FORMAT
AUTOTUNINGAUTOTUNING_ENABLEDAUTOTUNING_ENABLED_DEFAULTAUTOTUNING_FASTAUTOTUNING_FAST_DEFAULTAUTOTUNING_RESULTS_DIRAUTOTUNING_RESULTS_DIR_DEFAULTAUTOTUNING_EXPS_DIRAUTOTUNING_EXPS_DIR_DEFAULTAUTOTUNING_OVERWRITEAUTOTUNING_OVERWRITE_DEFAULTAUTOTUNING_START_PROFILE_STEP%AUTOTUNING_START_PROFILE_STEP_DEFAULTAUTOTUNING_END_PROFILE_STEP#AUTOTUNING_END_PROFILE_STEP_DEFAULTAUTOTUNING_METRIC_PATHAUTOTUNING_METRIC_PATH_DEFAULTAUTOTUNING_TUNER_TYPEAUTOTUNING_TUNER_GRIDSEARCHAUTOTUNING_TUNER_RANDOMAUTOTUNING_TUNER_MODELBASEDAUTOTUNING_TUNER_TYPE_DEFAULTAUTOTUNING_TUNER_EARLY_STOPPING'AUTOTUNING_TUNER_EARLY_STOPPING_DEFAULTAUTOTUNING_TUNER_NUM_TRIALS#AUTOTUNING_TUNER_NUM_TRIALS_DEFAULTAUTOTUNING_ARG_MAPPINGSAUTOTUNING_ARG_MAPPINGS_DEFAULTAUTOTUNING_MAX_TRAIN_BATCH_SIZE'AUTOTUNING_MAX_TRAIN_BATCH_SIZE_DEFAULTAUTOTUNING_MIN_TRAIN_BATCH_SIZE'AUTOTUNING_MIN_TRAIN_BATCH_SIZE_DEFAULT-AUTOTUNING_MAX_TRAIN_MICRO_BATCH_SIZE_PER_GPU5AUTOTUNING_MAX_TRAIN_MICRO_BATCH_SIZE_PER_GPU_DEFAULT-AUTOTUNING_MIN_TRAIN_MICRO_BATCH_SIZE_PER_GPU5AUTOTUNING_MIN_TRAIN_MICRO_BATCH_SIZE_PER_GPU_DEFAULT'AUTOTUNING_NUM_TUNING_MICRO_BATCH_SIZES/AUTOTUNING_NUM_TUNING_MICRO_BATCH_SIZES_DEFAULTAUTOTUNING_MP_SIZEAUTOTUNING_MP_SIZE_DEFAULTAUTOTUNING_METRICAUTOTUNING_METRIC_LATENCYAUTOTUNING_METRIC_THROUGHPUTAUTOTUNING_METRIC_FLOPSAUTOTUNING_METRIC_FORWARDAUTOTUNING_METRIC_BACKWRADAUTOTUNING_METRIC_STEPSAUTOTUNING_METRIC_DEFAULTAUTOTUNING_MODEL_INFO_PATH"AUTOTUNING_MODEL_INFO_PATH_DEFAULTMODEL_INFO_FORMAT
MODEL_INFOMODEL_INFO_PROFILEMODEL_INFO_PROFILE_DEFAULTMODEL_INFO_NUM_PARAMSMODEL_INFO_NUM_PARAMS_DEFAULTMODEL_INFO_HIDDEN_SIZEMODEL_INFO_HIDDEN_SIZE_DEFAULTMODEL_INFO_NUM_LAYERSMODEL_INFO_NUM_LAYERS_DEFAULTMODEL_INFO_KEY_DEFAULT_DICTDEFAULT_HF_CONFIGDEFAULT_MIN_MEM_CONFIGDEFAULT_TUNING_SPACE_ZERO_0DEFAULT_TUNING_SPACE_ZERO_1DEFAULT_TUNING_SPACE_ZERO_2DEFAULT_TUNING_SPACE_ZERO_3GLOBAL_TUNING_SPACETUNING_MICRO_BATCH_SIZE_PREFIX r   r   R/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/autotuning/constants.py<module>   s   		