o
    TiB                     @   sp   d dl mZmZmZmZ d dlmZmZ d dlm	Z	m
Z
 dZdZdZG dd deZd	d
 Zdd Zdd ZdS )   )	get_filesget_files_with_prefixpartition_dataget_zero_files)MODEL_FILE_PREFIXLAYER_FILE_PREFIX)reshape_meg_2d_parallelmeg_2d_parallel_mapPPTPDPc                   @   s@   e Zd ZdddZdddZdd Zd	d
 Zdd Zdd ZdS )model_3d_descr   c                 C   s   || _ || _|| _d S N	pp_degree	tp_degree	dp_degree)selfr   r   r    r   Y/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/checkpoint/reshape_3d_utils.py__init__   s   
zmodel_3d_desc.__init__Fc                 C   s`   |  |\}}|sJ d|t| j| j|j|j|d}t|| j| j | jd}t||jdS )N,)old_pp_degreeold_tp_degreenew_pp_degreenew_tp_degreeverbose)
meg_2d_mapsrc_2d_sizer   )r   r   )can_reshapejoinr   r   r   flatten_dp_dimensionr   unflatten_dp_dimension)r   target_3d_descr   valid_reshapereshape_errors
tgt_2d_mapflat_3d_mapr   r   r   reshape   s   
zmodel_3d_desc.reshapec                 C   s.   t  dt dt d| j d| j d| j dS )Nr   z = (z, ))PP_DIMTP_DIMDP_DIMr   r   r   r   r   r   r   get_desc'   s   .zmodel_3d_desc.get_descc                 C   s   | j | j | j S r   r   r-   r   r   r   
world_size*   s   zmodel_3d_desc.world_sizec           	      C   sd   g }d}|| j tf|| jtf|| jtffD ]\}}}||kr-d}|| d| d|  q||fS )NTFz indexing error: index z >= degree )r   r*   r   r+   r   r,   append)	r   pp_indextp_indexdp_indexerr_msgvalidindexdegreedim_namer   r   r   is_valid-   s   
zmodel_3d_desc.is_validc                 C   s   g }|j | j kr|dt d| j  d|j   |j| jkr.|dt d| j d|j  |j| jkrD|dt d| j d|j  t|dk|fS )Nz"Expansion reshape not supported - z: z --->     )r   r0   r*   r   r+   r   r,   len)r   r#   r4   r   r   r   r   8   s   zmodel_3d_desc.can_reshapeN)r   r   r   )F)	__name__
__module____qualname__r   r(   r.   r/   r9   r   r   r   r   r   r      s    

r   c                 C   s   t | }t| }tt|t d}|dkr-|}tt|t| }tdt|||  }ntt|t}tdt|| }d}t|||S )N01r:   r   )r   r   r;   r   r   r   maxr   )dir	file_listzero_file_listnum_pp0_filesr   r   r   r   r   r   get_model_3d_descriptorI   s   rE   c                    sn   t | j| j}t| jD ](}t| jD ] }| ||}|D ]  fddt|D }|||| qqq|S )Nc                    s   g | ]} |  qS r   r   ).0iidxr   r   r   
<listcomp>_       z(flatten_dp_dimension.<locals>.<listcomp>)r	   r   r   rangeget_dataadd_data)r   r   r   new_meg_2d_mapr1   r2   dp0_indicesdpX_indicesr   rH   r   r!   Y   s   r!   c           	         s|   | j  | j fddt|D }t D ]%}tD ]}| ||}t||}t||D ]\}}|||| q.qq|S )Nc                    s   g | ]}t  d qS )r   r   )r	   )rF   _rR   r   r   rJ   g   rK   z*unflatten_dp_dimension.<locals>.<listcomp>)r   r   rL   rM   r   ziprN   )	r   r   meg_2d_map_listr1   r2   flat_dp_indicespartitioned_dp_indices
dp_indices_2d_mapr   rR   r   r"   d   s   
r"   N)reshape_utilsr   r   r   r   	constantsr   r   reshape_meg_2dr   r	   r*   r+   r,   objectr   rE   r!   r"   r   r   r   r   <module>   s   8