o
    TiA                     @   s\   d dl Z ddlmZmZmZmZ ddlmZmZm	Z	 ddl
mZmZ dZG dd deZdS )	    N   )BASE_OPTIMIZER_STATEGROUP_PADDINGSOPTIMIZER_STATE_DICTPARTITION_COUNT)basic_folder_validationget_zero_filesmerge_state)model_3d_descget_model_3d_descriptorstatec                   @   s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	g dfddZ
d$ddZd$ddZdefddZdd Zdd Zdd Zd d! Zd"d# ZdS )%ZeROCheckpointc                 C   sv   t | || _t|| _t| j| _| jdksJ d| t|| _t| jj	| jj
| jjd| _| j| j| _d S )Nr   zNo ZeRO files found in )	pp_degree	tp_degree	dp_degree)r   dirr   	file_listlen	num_filesr   src_3dr
   r   r   r   	target_3dreshape_3d_file_map)selfr    r   X/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/checkpoint/zero_checkpoint.py__init__   s   

zZeROCheckpoint.__init__c                 C   s
   | j  S N)r   
world_sizer   r   r   r   get_src_world_size    s   
z!ZeROCheckpoint.get_src_world_sizec                 C      | j jS r   )r   r   r   r   r   r   get_src_tp_degree#      z ZeROCheckpoint.get_src_tp_degreec                 C   r!   r   )r   r   r   r   r   r   get_src_pp_degree&   r#   z ZeROCheckpoint.get_src_pp_degreec                 C   r!   r   )r   r   r   r   r   r   get_src_dp_degree)   r#   z ZeROCheckpoint.get_src_dp_degreec                 C   s>   |t | jk sJ d| dt | j | j| }|||S )Nz	DP index z >= DP degree )r   r   get_data)r   pp_indextp_indexdp_index	dp_2d_mapr   r   r   get_file_indices_for_rank,   s   (
z(ZeROCheckpoint.get_file_indices_for_rankc                    s      |||} fdd|D S )Nc                       g | ]} j | qS r   r   .0idxr   r   r   
<listcomp>3       z5ZeROCheckpoint.get_files_for_rank.<locals>.<listcomp>)r+   )r   r'   r(   r)   file_idx_listr   r   r   get_files_for_rank1   s   z!ZeROCheckpoint.get_files_for_rankTc                 C   s   |  |||}d }|D ]7}tj|tddd}	|D ]}
|	|
d  q|r*| |	 |d u r1|	}nt||	}| | |rB| | q|S )NcpuF)map_locationweights_only)	r4   torchloaddevicepop_strip_tensor_paddingsr	   _update_partition_count_clear_group_paddings)r   r'   r(   r)   keys_to_ignorestrip_tensor_paddingsstate_file_list	merged_sd
state_filesdkeyr   r   r   get_state_for_rank5   s    



z!ZeROCheckpoint.get_state_for_rankNc                 C   s:   |r	t d|  t| jD ]\}}|d|  qd S )Nz3D index map: zdp = )print	enumerater   
print_data)r   tagr)   _2d_mapr   r   r   print_3d_index_mapK   s
   z!ZeROCheckpoint.print_3d_index_mapc                    s~   |r	t d|  t jD ].\}}|jD ]&}|jD ] }|||} fdd|D }t | d| d| d|  qqqd S )Nz3D file map: c                    r,   r   r-   r.   r   r   r   r1   X   r2   z4ZeROCheckpoint.print_3d_file_map.<locals>.<listcomp>z, z => )rG   rH   r   r   r   r&   )r   rJ   r)   rK   r'   r(   file_index_listr   r   r   r   print_3d_file_mapQ   s   

 z ZeROCheckpoint.print_3d_file_maptarget_3d_descc                 C   s   || _ | j| j | _d S r   )r   r   r   r   )r   rO   r   r   r   r   [   s   zZeROCheckpoint.reshapec           	      C   s   |  |}|d u rd S | |t}|d u rd S | D ]6\}}|| dkr&q| D ]&\}}|dkrLt|rL| ||  }t|dd| ||< q*|||< q*qd S )Nr   step)	_get_param_group_states_get_optimizer_stater   itemsr8   	is_tensornumelnarrowclone)	r   rD   param_group_statesgroup_paddingsrE   group_state
state_namestate_value
raw_lengthr   r   r   r<   _   s    

z%ZeROCheckpoint._strip_tensor_paddingsc                 C   s2   |  |t}|rt|}dg| |t t< d S d S )Nr   )rR   r   r   r   )r   rD   rY   
num_groupsr   r   r   r>   r   s
   z$ZeROCheckpoint._clear_group_paddingsc                 C   s$   | td }|d u rd S | |d S r   )getr   )r   rD   	state_keyoptimizer_stater   r   r   rR   x   s   z#ZeROCheckpoint._get_optimizer_statec                 C   s<   | td }|d u rd S | td }|d u rd S | td S r   )r_   r   r   GROUP_STATE_KEY)r   rD   ra   base_optimizer_stater   r   r   rQ      s   z&ZeROCheckpoint._get_param_group_statesc                 C   s6   |  |t}|rt|}| jjg| |t t< d S d S r   )rR   r   r   r   r   r   )r   rD   partition_countsr^   r   r   r   r=      s
   z&ZeROCheckpoint._update_partition_countr   )__name__
__module____qualname__r   r    r"   r$   r%   r+   r4   rF   rL   rN   r
   r   r<   r>   rR   rQ   r=   r   r   r   r   r      s"    


r   )r8   	constantsr   r   r   r   reshape_utilsr   r   r	   reshape_3d_utilsr
   r   rb   objectr   r   r   r   r   <module>   s   