o
    TÃi  ã                   @   s@   d dl Z d dlmZ ddlT G dd„ deƒZG dd„ deƒZdS )	é    N)Úloggeré   )Ú*c                   @   s&   e Zd Zdd„ Zddd„Zdd„ ZdS )	ÚBaseSchedulerc                 C   s
   i | _ d S ©N©Ústate©Úself© r   új/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/data_pipeline/data_routing/scheduler.pyÚ__init__   ó   
zBaseScheduler.__init__Nc                 C   s~   | j t }|d u r|d }t|ƒ|t  d|  }t || j t | j t   | j t  ¡}|||t  8 }t	|| j t ƒ}|S )NÚroot_degreeg      ð?)
r   ÚRANDOM_LTD_SCHEDULE_CONFIGÚfloatÚRANDOM_LTD_REQUIRE_STEPÚmathÚfloorÚRANDOM_LTD_MAX_VALUEÚRANDOM_LTD_MIN_VALUEÚRANDOM_LTD_INCREASE_STEPÚmin)r
   Úglobal_stepsr   Ús_stateÚnext_seqr   r   r   Ú__fixed_root_get_value   s   
ÿz$BaseScheduler.__fixed_root_get_valuec                 C   s"   | j t dkr|  |d¡S tdƒ‚)NÚfixed_linearé   z$Unsupported random LTD schedule type)r   ÚRANDOM_LTD_SCHEDULER_TYPEÚ$_BaseScheduler__fixed_root_get_valueÚRuntimeError©r
   r   r   r   r   Ú	get_value   s   zBaseScheduler.get_valuer   )Ú__name__Ú
__module__Ú__qualname__r   r    r#   r   r   r   r   r      s    
r   c                       sl   e Zd Z‡ fdd„Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Z‡  ZS )ÚRandomLTDSchedulerc                    sb   t ƒ  ¡  |t | _|t | _|t | _|t | _	|  
¡  |t t r*t d¡ t‚d| jt< d S )Nz&**********Work In Progress************r   )Úsuperr   ÚRANDOM_LTD_TOTAL_LAYER_NUMÚmodel_layer_numÚRANDOM_LTD_LAYER_NUMÚrandom_ltd_layer_numÚRANDOM_LTD_SCHEDULERÚconfig_scheduleÚRANDOM_LTD_GLOBAL_BATCH_SIZEÚglobal_batch_sizeÚreset_to_initÚ"RANDOM_LTD_LAYER_TOKEN_LR_SCHEDULEÚ!RANDOM_LTD_LAYER_TOKEN_LR_ENABLEDr   ÚwarningÚNotImplementedErrorr   Ú RANDOM_LTD_CONSUMED_LAYER_TOKENS)r
   Úconfig©Ú	__class__r   r   r   (   s   





zRandomLTDScheduler.__init__c                 C   s"   t |ƒD ]}|  |¡ q| jt S r   )ÚrangeÚ
update_seqr   r6   )r
   Útrain_itersÚstepr   r   r   Úget_total_layer_tokens7   s   
z)RandomLTDScheduler.get_total_layer_tokensc                 C   sr   | j d ur-| j t | jt< | j t | jt< | j t | jt< | j t | jt< | j t | jt< d| jt< d| jt< d S )Nr   éÿÿÿÿ)	r.   r   r   r   ÚRANDOM_LTD_CURRENT_VALUEr   r   r6   ÚRANDOM_LTD_CURR_STEPr	   r   r   r   r1   <   s   

z RandomLTDScheduler.reset_to_initc                 C   s
   | j t S r   ©r   r@   r	   r   r   r   Úget_current_seqF   r   z"RandomLTDScheduler.get_current_seqc                 C   s   || j t< d S r   rB   )r
   Ú
seq_lengthr   r   r   Úset_current_seqI   s   z"RandomLTDScheduler.set_current_seqc                 C   ó   | j S r   )r,   r	   r   r   r   Úget_random_ltd_layer_numL   ó   z+RandomLTDScheduler.get_random_ltd_layer_numc                 C   rF   r   r   r	   r   r   r   Ú	get_stateO   rH   zRandomLTDScheduler.get_statec                 C   s
   || _ d S r   r   )r
   r   r   r   r   Ú	set_stateR   r   zRandomLTDScheduler.set_statec                 C   s~   | j t | j t k r|  |¡| j t< || j t kr=| j t  | j| j t | j | j t | j| j    7  < || j t< d S d S r   )	r   r@   r   r#   rA   r6   r0   r,   r*   r"   r   r   r   r;   U   s   
ÿýzRandomLTDScheduler.update_seqc                 C   s6   t | jt  t| jt t| jt t| jt t| jt iS r   ©r6   r   rA   r@   r   r   r	   r   r   r   Ú
state_dict]   s   




ûzRandomLTDScheduler.state_dictc                 C   sJ   |t  | jt < |t | jt< |t | jt< |t | jt< |t | jt< d S r   rK   )r
   rL   r   r   r   Úload_state_dictf   s
   z"RandomLTDScheduler.load_state_dict)r$   r%   r&   r   r>   r1   rC   rE   rG   rI   rJ   r;   rL   rM   Ú__classcell__r   r   r8   r   r'   &   s    
	r'   )r   Údeepspeed.utilsr   Ú	constantsÚobjectr   r'   r   r   r   r   Ú<module>   s
   