o
    Ti                     @   s   d dl Z d dlZd dlZd dlZd dlmZmZmZ d dlm	Z	 ddl
mZmZmZmZmZmZ e	G dd dZdd	 Zd
d ZdS )    N)ListTupleUnion)	dataclass   )FP32_WEIGHT_KEYPARAMVOCAB_TENSORCAT_DIMPARAM_N_SUB_PARAMSSUB_PARAM_SHAPEc                   @   s:   e Zd ZU ee ed< eeee ef  ed< eed< dS )SubparamShapepatternsshapepartition_dimN)	__name__
__module____qualname__r   str__annotations__r   r   int r   r   ]/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/checkpoint/universal_checkpoint.pyr      s   
 r   c           !   
      s  | j }i |_g }t|D ]}d}t||}|r!||d qd }	|D ]`}
tj	||
 d}t
j|dd}|
dkrA|}	q&|t }|j| jkrOdd|td}|r| jd  }||jd ksrJ d| d	|jd  ||jd kr||jd  }t
jj|ddd|fd
d}| }|  }|| ksJ d| d| d| d |td }|td |td}|r|j}|j| }t|ts|f}dd |jD }||}d}g }|D ]}||||j|d }|| ||7 }qt
j||d}n'|dkr%|j| d} fdd|D }t
j| d}n|  }| }|j}|d|j |j}|
t!krf|" } |  |jks^J d|
 d|   d|j | j#$|j# q&| |jks~J d|
 d|  d|j |% & |j|
< q&|	S )Nz(.+).ptr   z.ptF)weights_onlystepr   zVocab tensor padded size z < loaded universal size constantzLoading z full param numel z != tensor slice numel z * tp_world_size c                 S   s"   g | ]}t |trt|n|qS r   )
isinstancetuplesum).0dr   r   r   
<listcomp>d   s   " z,load_hp_checkpoint_state.<locals>.<listcomp>dimc                    s   g | ]}|j  d  qS )r"   )chunk)r   p	chunk_dimtp_ranktp_world_sizer   r   r!   s   s    zLoad checkpoint z dst numel z != src numel )'_hp_mappingoptim_fragmentoslistdirresearchappendgrouppathjointorchloadr   r   getr	   nn
functionalpadnumelr   r
   r   r   r   r   viewnarrowr$   catflattenlp_fragment_addressstartr   get_hp_fragmentdatacopy_clonedetach)!selffolderr(   r)   
hp_mappinghp_keysfilepatternmatchr   key	ckpt_file	ckpt_dictfull_hp_paramis_vocab_tensorpadded_target_vocab_sizepadding_sizefull_param_numeltp_slice_numelsub_param_shapen_sub_paramsr   sub_dim_sizespartition_shapeoffsetmerged_chunkssub_dim_sizesub_params_tp_slicetp_hp_slice
sub_paramslp_frag_addresstp_hp_fragment
dst_tensorr   r&   r   load_hp_checkpoint_state   s   







rc   c                 C   s   | D ]	}t t||_qd S )N)types
MethodTyperc   )
param_listparamr   r   r   enable_universal_checkpoint   s   rh   )r,   r.   r4   rd   typingr   r   r   dataclassesr   	constantsr   r   r	   r
   r   r   r   rc   rh   r   r   r   r   <module>   s    z