o
    Tig
                     @   sX   d dl Z d dlZd dlmZ d dlmZ d dlmZ G dd deZ	G dd de	Z
dS )	    N)logger)map_to_flat_opt_states)bwc_tensor_model_parallel_rankc                   @   s   e Zd ZdS )DeepSpeedOptimizerN)__name__
__module____qualname__ r	   r	   T/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/base_optimizer.pyr      s    r   c                   @   s"   e Zd ZdededdfddZdS )ZeROOptimizerlp_groups_namecheckpoint_dirreturnNc              	      s  t j|d}t j|d}t j|sJ | dtj|dd}| | t| jd}| jd u r:t	
d d}nt| jd	rE| j n| j }tt| jj|d
 D ]\}\}}	t }
g  t| |}|| D ](}|jd ur|t j|| j| ||}|j D ]}|
| q | qi|d d }t fdd D sJ d  d d d ur d | jj| d< t||| | jj|
 |	 D ]\}}|dkrq|||< qqUd S )Nzerozoptimizer_state.ptz> containing optimizer global state is missing! Cannot proceed.F)weights_only)mpuz@MPU is not provided, setting tp size to 1 in checkpoint loading.   get_slice_parallel_world_sizeparam_groupsparamsr   c                 3   s    | ]	}| d  kV  qdS )r   Nr	   ).0stepstepsr	   r
   	<genexpr>6   s    zMZeROOptimizer.load_hp_checkpoint_state_from_checkpoint_dir.<locals>.<genexpr>zSteps z are not equalr   )ospathjoinisfiletorchload_load_global_stater   r   r   warninghasattrr   $get_tensor_model_parallel_world_size	enumeratezip	optimizerr   setgetattr_hp_mappingload_hp_checkpoint_stateparam_namesget_optim_state_keysaddappendallstater   items)selfr   r   optim_state_pathoptim_sdtp_ranktp_world_sizeiparam_grouploaded_param_groupopt_keys	lp_groupslpr   keyhp_paramvaluer	   r   r
   ,load_hp_checkpoint_state_from_checkpoint_dir   sT   





&
z:ZeROOptimizer.load_hp_checkpoint_state_from_checkpoint_dir)r   r   r   strrA   r	   r	   r	   r
   r      s    r   )r   r   deepspeed.utilsr   deepspeed.utils.tensor_fragmentr   deepspeed.runtime.utilsr   objectr   r   r	   r	   r	   r
   <module>   s   