o
    ÓÙ¾iî  ã                   @   s`   d Z ddlZddlZddlZddlZddlZddlZddlmZm	Z	 e 
e¡ZG dd„ dƒZdS )z¡ Checkpoint Saver

Track top-n training checkpoints and maintain recovery checkpoints on specified intervals.

Hacked together by / Copyright 2020 Ross Wightman
é    Né   )Úunwrap_modelÚget_state_dictc                
   @   sj   e Zd Zdddddddddef
dd„Zd	d
„ Zdd„ Zddd„Zddd„Zddd„Z	ddd„Z
dd„ ZdS )ÚCheckpointSaverNÚ
checkpointÚrecoveryÚ Fé
   c                 C   s–   || _ || _|| _|| _|| _g | _d | _d | _d| _d| _	d| _
|| _|	| _|| _|| _d| _|
| _|
r8tjntj| _|| _|| _| jdksIJ ‚d S )Nr   Tz.pth.tarr   )ÚmodelÚ	optimizerÚargsÚ	model_emaÚ
amp_scalerÚcheckpoint_filesÚ
best_epochÚbest_metricÚcurr_recovery_fileÚprev_recovery_fileÚcan_hardlinkÚcheckpoint_dirÚrecovery_dirÚsave_prefixÚrecovery_prefixÚ	extensionÚ
decreasingÚoperatorÚltÚgtÚcmpÚmax_historyÚ	unwrap_fn)Úselfr
   r   r   r   r   Úcheckpoint_prefixr   r   r   r   r   r    © r#   úO/home/ubuntu/.local/lib/python3.10/site-packages/timm/utils/checkpoint_saver.pyÚ__init__   s*   zCheckpointSaver.__init__c              
   C   s`   | j r(ztj |¡rt |¡ W n ttfy' } z	d| _ W Y d }~nd }~ww t ||¡ d S ©NF)r   ÚosÚpathÚexistsÚunlinkÚOSErrorÚNotImplementedErrorÚreplace©r!   ÚsrcÚdstÚer#   r#   r$   Ú_replaceB   s   
€€ÿzCheckpointSaver._replacec              
   C   sn   | j r/ztj |¡rt |¡ t ||¡ W d S  ttfy. } z	d| _ W Y d }~nd }~ww t 	||¡ d S r&   )
r   r'   r(   r)   r*   Úlinkr+   r,   ÚshutilÚcopy2r.   r#   r#   r$   Ú
_duplicateK   s   
€ÿzCheckpointSaver._duplicatec                 C   s¤   |t | jƒj ¡ t| j| jƒ| j ¡ ddœ}| jd ur&| jj|d< | j|d< | j	d ur4| j	 ¡ || j	j
< | jd urBt| j| jƒ|d< |d urJ||d< t ||¡ d S )Né   )ÚepochÚarchÚ
state_dictr   Úversionr9   r   Ústate_dict_emaÚmetric)Útyper
   Ú__name__Úlowerr   r    r   r:   r   r   Ústate_dict_keyr   ÚtorchÚsave)r!   Ú	save_pathr8   r=   Ú
save_stater#   r#   r$   Ú_saveW   s    û



zCheckpointSaver._saver   c                 C   s¶   t t| jƒ|ƒ}| j| }|dk st| jƒ|krd S | j|d … }|D ]-}zt d |¡¡ t |d ¡ W q# t	yP } zt 
d |¡¡ W Y d }~q#d }~ww | jd |… | _d S )Nr   zCleaning checkpoint: {}z(Exception '{}' while deleting checkpoint)ÚminÚlenr   r   Ú_loggerÚdebugÚformatr'   ÚremoveÚ	ExceptionÚerror)r!   ÚtrimÚdelete_indexÚ	to_deleteÚdr1   r#   r#   r$   Ú_cleanup_checkpointsj   s   
€ÿz$CheckpointSaver._cleanup_checkpointsc                 C   s˜  |dksJ ‚t j | jd| j ¡}t j | jd| j ¡}|  |||¡ |  ||¡ | jr1| jd nd }t| jƒ| j	k sG|d u sG|  
||d ¡r¿t| jƒ| j	krT|  d¡ d | jt|ƒg¡| j }t j | j|¡}|  ||¡ | j ||f¡ t| jdd„ | j d	| _d
}| jD ]	}	|d |	¡7 }q‰t |¡ |d ur¿| jd u s¨|  
|| j¡r¿|| _|| _t j | jd| j ¡}
|  ||
¡ | jd u rÆdS | j| jfS )Nr   ÚtmpÚlastéÿÿÿÿr   ú-c                 S   s   | d S )Nr   r#   )Úxr#   r#   r$   Ú<lambda>Ž   s    z1CheckpointSaver.save_checkpoint.<locals>.<lambda>)ÚkeyÚreversezCurrent checkpoints:
z {}
Ú
model_best)NN)r'   r(   Újoinr   r   rF   r2   r   rH   r   r   rS   r   Ústrr6   ÚappendÚsortedr   rK   rI   Úinfor   r   )r!   r8   r=   Útmp_save_pathÚlast_save_pathÚ
worst_fileÚfilenamerD   Úcheckpoints_strÚcÚbest_save_pathr#   r#   r$   Úsave_checkpointx   s>   ÿ
ý

 zCheckpointSaver.save_checkpointc              
   C   sæ   |dksJ ‚t j | jd| j ¡}|  ||¡ d | jt|ƒt|ƒg¡| j }t j | j|¡}|  ||¡ t j 	| j
¡rjzt d | j
¡¡ t  | j
¡ W n tyi } zt d || j
¡¡ W Y d }~nd }~ww | j| _
|| _d S )Nr   Úrecovery_tmprW   zCleaning recovery: {}z Exception '{}' while removing {})r'   r(   r]   r   r   rF   r   r^   r2   r)   r   rI   rJ   rK   rL   rM   rN   r   )r!   r8   Ú	batch_idxrb   re   rD   r1   r#   r#   r$   Úsave_recoveryŸ   s      €ÿ
zCheckpointSaver.save_recoveryc                 C   sB   t j | j| j¡}t |d | j ¡}t|ƒ}t|ƒr|d S dS )NÚ*r   r   )	r'   r(   r]   r   r   Úglobr   r`   rH   )r!   Úrecovery_pathÚfilesr#   r#   r$   Úfind_recovery±   s   zCheckpointSaver.find_recovery)N)r   )r?   Ú
__module__Ú__qualname__r   r%   r2   r6   rF   rS   ri   rl   rq   r#   r#   r#   r$   r      s&    
ó+	



'r   )Ú__doc__rn   Úloggingr   r'   r4   rB   r
   r   r   Ú	getLoggerr?   rI   r   r#   r#   r#   r$   Ú<module>   s    
