o
    Ti                     @   s   d dl Z d dlZd dlZd dlmZ ddlmZmZmZm	Z	 dd Z
dd Zd	d
 Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zg fddZdS )    N)OrderedDict   )ZERO_FILE_PREFIXFP16_ZERO_FILE_PREFIXBF16_ZERO_FILE_PREFIXMODEL_FILE_PREFIXc                 C   s8   t j| sJ |  dt j| sJ |  dd S )Nz path does not existz is not a folder)ospathexistsisdir)dir r   V/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/checkpoint/reshape_utils.pybasic_folder_validation   s   r   c                 C   s:   g }| D ]}t j|\}}||r|| qt|S N)r   r	   split
startswithappendsorted)	all_filesprefix	file_list	file_path_fnamer   r   r   get_files_with_prefix   s   

r   c                 C   s*   | D ]}t j|std| d qd S )NzError: z is not existent)r   r	   isfileprint)r   filer   r   r   validate_files   s
   r   c                 C   s<   g }t | D ]\}}}|D ]}|t j|| qq|S r   )r   walkr   r	   join)r   r   rootr   filesr   r   r   r   	get_files"   s   r$   c           	      C   s   | dt  d}g }| D ]'}t||}|r-t|d}t|d}||||f qtd| t|dd d}d	d
 |D S )Nz	([0-9]+)_z([0-9]+)r      z&Cannot parse dp_rank and mp_rank from c                 S   s   | d | d fS )Nr   r   r   )xr   r   r   <lambda>6   s    z!sort_zero_files.<locals>.<lambda>)keyc                 S   s   g | ]\}}}|qS r   r   ).0r   fr   r   r   
<listcomp>7   s    z#sort_zero_files.<locals>.<listcomp>)r   researchintgroupr   
ValueErrorr   )	r#   r   pattern
rank_pairsr*   mdp_rankmp_ranksorted_filesr   r   r   sort_zero_files*   s   r7   c                 C   s@   t | }tttfD ]}t||}t|dkrt||  S q	g S )Nr   )r$   r   r   r   r   lenr7   )r   r   r   
zero_filesr   r   r   get_zero_files:   s   
r:   c                    s@   t  }|| dksJ ||  fddtd|D }|S )Nr   c                    s   g | ]
} ||  qS r   r   )r)   i	data_listpartition_sizer   r   r+   H       z"partition_data.<locals>.<listcomp>)r8   range)r=   num_partitions	num_elemspartitions_listr   r<   r   partition_dataD   s
   rD   c                 C   s
   d | S )N.)r!   key_listr   r   r   _key_list_to_stringL   s   
rH   c                 C   sV   t | i }| D ]\}}||  v r$t| | || t|g||< q
|||< q
|S r   )typeitemskeysmerge_statestr)dict_adict_brG   merged_dictr(   valuer   r   r   merge_state_dictP   s    
rR   c                    sR   t | t |krtt   tdt |  dt |  fddt| |D S )Nz-Cannot merge lists of different lengths, a = z b = c                    s   g | ]
\}}t || qS r   )rL   )r)   abrF   r   r   r+   a   r?   z$merge_state_list.<locals>.<listcomp>)r8   r   rH   r0   zip)list_alist_brG   r   rF   r   merge_state_list\   s   rX   c                 C   s   t | t |kr!t|}td|  tdt |  dt | t | ttfv r/t| ||S t | ttfv rAt | t	| ||S t
| rNt
| |gdS | S )Nzkey_list = z!Cannot merge two states of types z
 and type r   )rI   rH   r   r0   dictr   rR   listtuplerX   torch	is_tensorcat)state_astate_brG   key_list_stringr   r   r   rL   d   s   
rL   )r   r,   r\   collectionsr   	constantsr   r   r   r   r   r   r   r$   r7   r:   rD   rH   rR   rX   rL   r   r   r   r   <module>   s    

