o
    Ti)                     @   s   d dl T ddlZddlmZ dd Zdd	 Zd
d Zdd Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ ZdS ),   )*    N   )get_scalar_paramc                 C   sT   i }t | |t< t| |t< t|  vri | t< | t }t||t< t||t	< |S N)
get_data_efficiency_enabledDATA_EFFICIENCY_ENABLEDget_data_efficiency_seedDATA_EFFICIENCY_SEEDDATA_EFFICIENCYkeysget_data_samplingDATA_SAMPLINGget_data_routingDATA_ROUTING
param_dictoutputsub_param_dict r   Z/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/data_pipeline/config.pyget_data_efficiency_config   s   r   c                 C       t |  v rt| t  ttS dS NF)r   r   r   r   DATA_EFFICIENCY_ENABLED_DEFAULTr   r   r   r   r         r   c                 C       t |  v rt| t  ttS tS r   )r   r   r   r
   DATA_EFFICIENCY_SEED_DEFAULTr   r   r   r   r	   !   r   r	   c                 C   sb   |  ti }t|}t| |t< t| |t< t| |t< t	| |t
< t||t< t||t< |S r   )getr   copyget_data_sampling_enabledDATA_SAMPLING_ENABLEDget_data_sampling_num_epochsDATA_SAMPLING_NUM_EPOCHSget_data_sampling_num_workersDATA_SAMPLING_NUM_WORKERSget_data_sampling_pin_memoryDATA_SAMPLING_PIN_MEMORYget_curriculum_learningCURRICULUM_LEARNINGget_dynamic_batchingDYNAMIC_BATCHING)r   r   r   r   r   r   r   (   s   
r   c                 C   r   r   )r   r   r   r"   DATA_SAMPLING_ENABLED_DEFAULTr   r   r   r   r!   4   r   r!   c                 C   r   r   )r   r   r   r$    DATA_SAMPLING_NUM_EPOCHS_DEFAULTr   r   r   r   r#   ;   r   r#   c                 C   r   r   )r   r   r   r&   !DATA_SAMPLING_NUM_WORKERS_DEFAULTr   r   r   r   r%   B   
   
r%   c                 C   r   r   )r   r   r   r(    DATA_SAMPLING_PIN_MEMORY_DEFAULTr   r   r   r   r'   J   r   r'   c                 C   sr   i }t | |t< t|  vri | t< | t }|t r7t| v s(J dt dt|  D ]\}}|||< q.|S )Nz Curriculum learning is enabled,  must be specified)get_curriculum_learning_enabledCURRICULUM_LEARNING_ENABLEDr*   r   CURRICULUM_LEARNING_METRICSget_curriculum_learning_paramsitems)r   r   r   keyvalr   r   r   r)   Q   s   

r)   c                 C   s   t  | ti }t|tt|t< t|tt|t< t	|t
t|t
< t| v r3t	|t nt|t< t|tt|t< |t r[t| v sSJ dt dt	|t |t< t|td|t< |S )Nz Dynamic batching is enabled, so r2   F)r    r   r,   boolDYNAMIC_BATCHING_ENABLED DYNAMIC_BATCHING_ENABLED_DEFAULTstr"DYNAMIC_BATCHING_LR_SCALING_METHOD*DYNAMIC_BATCHING_LR_SCALING_METHOD_DEFAULTintDYNAMIC_BATCHING_MIN_BATCH_SIZE'DYNAMIC_BATCHING_MIN_BATCH_SIZE_DEFAULTDYNAMIC_BATCHING_MAX_BATCH_SIZEr   'DYNAMIC_BATCHING_MAX_BATCH_SIZE_DEFAULT'DYNAMIC_BATCHING_SEQUENCE_PICKING_ORDER/DYNAMIC_BATCHING_SEQUENCE_PICKING_ORDER_DEFAULTDYNAMIC_BATCHING_MAX_TOKENSDYNAMIC_BATCHING_VERBOSE)r   r   r   r   r   r+   _   s,   



r+   c                 C   r   r   )r*   r   r   r4   #CURRICULUM_LEARNING_ENABLED_DEFAULTr   r   r   r   r3   s   r0   r3   c                 C   ,   t |  v rt| t  }|t |S i S r   )r*   r   r    popr4   )r   curriculum_learning_paramsr   r   r   r6   {   
   
r6   c                 C   r   r   )CURRICULUM_LEARNING_LEGACYr   r   CURRICULUM_ENABLED_LEGACY!CURRICULUM_ENABLED_DEFAULT_LEGACYr   r   r   r   get_curriculum_enabled_legacy   r0   rQ   c                 C   s,   t |  v rt| t  }|t |S dS r   )rN   r   r    rK   rO   )r   curriculum_paramsr   r   r   get_curriculum_params_legacy   rM   rS   c                 C   s<   i }t | |t< t|  vri | t< | t }t||t< |S r   )get_data_routing_enabledDATA_ROUTING_ENABLEDr   r   get_random_ltd
RANDOM_LTDr   r   r   r   r      s   r   c                 C   r   r   )r   r   r   rU   DATA_ROUTING_ENABLED_DEFAULTr   r   r   r   rT      r   rT   c                 C   sV   i }t |t< i |t< t|t t< t| r)t| |t< t|  D ]\}}|||< q |S r   )RANDOM_LTD_ENABLED_DEFAULTRANDOM_LTD_ENABLED"RANDOM_LTD_LAYER_TOKEN_LR_SCHEDULE)RANDOM_LTD_LAYER_TOKEN_LR_ENABLED_DEFAULT!RANDOM_LTD_LAYER_TOKEN_LR_ENABLEDget_random_ltd_enabledget_random_ltd_paramsr7   )r   r   r8   r9   r   r   r   rV      s   
rV   c                 C   r   r   )rW   r   r   rZ   rY   r   r   r   r   r^      r   r^   c                 C   rJ   r   )rW   r   r    rK   rZ   )r   random_ltd_paramsr   r   r   r_      rM   r_   )	constantsr    config_utilsr   r   r   r	   r   r!   r#   r%   r'   r)   r+   r3   r6   rQ   rS   r   rT   rV   r^   r_   r   r   r   r   <module>   s,   		