o
    i                     @   s   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dl	m
Z
 d dlZd dlmZ d dlZd dlZd dlZd dlmZ d d	lmZ ddedefddZe ddedefddZdS )    N)Path)Optional)Sequence)Union)BytesIO)
Collection)OrderedDict)
cmp_to_key   F
output_dirlast_nc                    s6  zk|st jtj ddd}nt jtj dddd}|d }|d| d }t| dd	 d
d}|dkr>|d| n|| d }g }|d| D ]\}	}
|s[tj |	}ntj |	d}|| qMW |S    t| d t	 }dd |D }|j
dd	 d
d  fdd|d| D }Y |S )zj
    Get the paths of the last 'last_n' checkpoints by parsing filenames
    in the output directory.
    zmodel.ptcpumap_locationzmp_rank_00_model_states.ptavg_keep_nbest_models_typeval__step_or_epochc                 S   s   | d S )N    xr   r   [/home/ubuntu/.local/lib/python3.10/site-packages/funasr/train_utils/average_nbest_models.py<lambda>"   s    z'_get_checkpoint_paths.<locals>.<lambda>T)keyreverseaccNz+ does not exist, avg the lastet checkpoint.c                 S   s   g | ]	}| d r|qS )z
model.pt.e)
startswith.0fr   r   r   
<listcomp>3       z)_get_checkpoint_paths.<locals>.<listcomp>c                 S   s   t td|  S )Nz(\d+))intresearchgroupr   r   r   r   r   5   s    c                    s   g | ]	}t j |qS r   )ospathjoinr   r   r   r   r    7   r!   )torchloadr&   r'   r(   sorteditemsappendprintlistdirsort)r   r   use_deepspeedkwargs
checkpointr   val_step_or_epochsorted_itemscheckpoint_pathsr   valueckptfilescheckpoint_filesr   r)   r   _get_checkpoint_paths   s6    
r<   c                    s  t | |fi |}td|  g }|D ]}tj|r)|tj|ddd  qtd| d qt|dk r>td d	S t	 }|d
 
 D ],  fdd|D }t|d
 jdret|}|| < qGt|}	tj|	d
d| < qGtj| d| }
td|i|
 |
S )z
    Average the last 'last_n' checkpoints' model state_dicts.
    If a tensor is of type torch.int, perform sum instead of average.
    zaverage_checkpoints: r   r   
state_dictzCheckpoint file z not found.r   z#No checkpoints found for averaging.Nr   c                    s   g | ]}|    qS r   )r   )r   r=   r   r   r   r    U   s    z'average_checkpoints.<locals>.<listcomp>z	torch.int)dimzmodel.pt.avg)r<   r/   r&   r'   isfiler.   r*   r+   lenr   keysstrdtyper   sumstackmeanr(   save)r   r   r3   r7   state_dictsr'   avg_state_dicttensorssummed_tensorstacked_tensorscheckpoint_outpathr   r>   r   average_checkpoints<   s*   

rO   )r
   F)r
   )loggingpathlibr   typingr   r   r   warningsr&   ior   r*   r   r#   collectionsr   	functoolsr	   rC   r"   r<   no_gradrO   r   r   r   r   <module>   s$    )