o
    oi(                     @   s  d dl Z d dlZd dlZd dlmZmZmZmZ d dlZ	d dl
Z
d dlmZ d dl
mZ d dlmZmZ d dlmZ d dlmZ d dlmZ d	efd
dZ					d3dee dedededededeeedf d	eejef fddZddg dfdee
jjejf dededeeedf def
ddZ			d4dee
jjejf dededed ee  f
d!d"Z!d5dededefd$d%Z"	d6ded&ed'e d(ed)ef
d*d+Z#ded	eee e f fd,d-Z$ded.ed e fd/d0Z%d1d2 Z&dS )7    N)ListOptionalTupleUnion)logger)nn)Csvconfig)
init_model)check_finite_module)DFreturnc                 C   s$   t tj| dd dd S )N.r   _)intospathbasenamesplit)cp r   K/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/df/checkpoint.py	get_epoch   s   $r   Fckptlatestcp_dirdf_statejit	mask_onlytrain_df_only	extensionepochc           	      C   s   |r|rt dt||du |du d}|rtj|}tdg t ddd}| d ur?t|d| |||d}|d u r9d	n|}||fS d	}||fS )
Nz6Only one of `mask_only` `train_df_only` can be enabledF)run_df
train_maskCP_BLACKLISTtrain)savesectionmodel)	blacklistr!   r"   r   )
ValueErrorr
   torchr   scriptr	   r   read_cp)	r   r   r   r   r    r!   r"   r)   r*   r   r   r   
load_model   s   	r/   Tobjnamedirnamelogc              
      s  g }t trdv sJ dkr,ttj|| d| d}t|dkr,td t|dkrSttj|| d| }|ttj|| d| d7 }t|dkr[d S t t	r}t
fdd|D d }|d u r|td	  td
 n
t|td}t||rtd| tj|dd}dd | D }|rtddd |D d d t|}	fdd| D }t||	k rtd| t | tjrA	 z| j|dd\}
}W n. ty } z!t| t  d v r fdd| D }W Y d }~q|d }~ww 	 |
D ]}td| d q|D ]}|ds3d|v r5q&td|  q&S | | d S ) N)bestr   r4   *..bestr   z9Could not find `best` checkpoint. Checking for default...c                 3   s     | ]}t | kr|V  qd S N)r   ).0x)r"   r   r   	<genexpr>D   s    zread_cp.<locals>.<genexpr>z#Could not find checkpoint of epoch    )keyz!Found checkpoint {} with epoch {}cpu)map_locationc                 S   s   i | ]\}}| d d|qS )clcdf)replacer8   kvr   r   r   
<dictcomp>N       zread_cp.<locals>.<dictcomp> c                 s   s    | ]	}d | dV  qdS )(z)|Nr   )r8   br   r   r   r:   P   s    r   c                    s$   i | ]\}}  |d u r||qS r7   )searchrB   )regr   r   rE   R   s   $ zFiltered checkpoint modules: {}TF)strictzsize mismatchc                    s   i | ]\}}| vr||qS r   r   rB   )e_strr   r   rE   ]   rF   zMissing key: ''z.h0erb_compzUnexpected key: )
isinstancestrglobr   r   joinlenr   warningr   nexterrorexitmaxr   infoformatr,   loaditemsrecompiler   Moduleload_state_dictRuntimeErrorendswith)r0   r1   r2   r"   r!   r*   r3   checkpointsr   
len_beforemissing
unexpecteder<   r   )rM   r"   rK   r   r.   .   sh   	
 
$
"

r.   minmetricc              
   C   s  t |  tddtdd}tddtdd}|d ur|dv sJ t|}ttj|dd	}	|	d
 |		 }
t
|
d
krGt|dkrCdnd}nt|
d  dd }|dkrZdnd}t|||rtd| d|  |	d
tj t|	tt||gg tj|| d| d| d}t|  | t|||d |d W d    n1 sw   Y  tj|| d| d| }td| d|  t|  | t||||d d S )Nn_checkpoint_history   r&   )defaultcastr(   n_best_checkpoint_history   )ri   rY   r6   za+r   ri   infz-infr    r;   __lt____gt__z$Saving new best checkpoint at epoch z with metric: r   r   )nkeepzWriting checkpoint z with epoch )r   r	   r   floatopenr   r   rS   seek	readlinesrT   stripr   getattrr   rZ   SEEK_ENDnpsavetxtarrayr,   r'   
state_dictcleanup)r0   r1   r2   r"   r!   rj   cmpn_keepn_keep_bestprev_best_flines	prev_bestcp_namer   r   r   write_cpk   s4   	
 r   rp   c                 C   sx   |dk rd S t  tj||  d| }t|dkrd S t|tdd}||d  D ]}td	| t
| q*d S )Nr   r5   T)r<   reversezRemoving old checkpoint: {})rR   r   r   rS   rT   sortedr   r   debugr[   remove)r1   r2   r!   ru   rd   r   r   r   r   r      s   r   max_patience
new_metricr   raise_c           	      C   s   |dkrdnd}t |}t| \}}|d u st|||r%t| d| dS |d }t| || ||krNd|dd|dd	| d
}|rGt|t| dS dS )Nri   rs   rt   r   r;   z&No improvements on validation metric (z.3fz - z) for z epochs. Stopping.FT)rv   read_patiencer{   write_patiencer+   r   rU   )	r2   r   r   r   r   prev_patienceprev_metricnew_patiencemsgr   r   r   check_patience   s   
r   c                 C   s<   t j| d}t j|sdS t|\}}t|t|fS )N	.patience)Ng        )r   r   rS   isfiler}   loadtxtr   rv   )r2   fnpatiencerj   r   r   r   r      s
   r   r   c                 C   s   t tj| d||gS )Nr   )r}   r~   r   r   rS   )r2   r   rj   r   r   r   r      s   r   c                  C   sd  dd l } |  &}t|dd t|dd t|dd t|dddddu s'J W d    n1 s1w   Y  |  ,}t|dd t|dd t|dd t|dd t|dddddu s_J W d    n1 siw   Y  |  6}t|dddd t|dd	dd t|dddd t|dddd t|ddddd
du sJ W d    d S 1 sw   Y  d S )Nr   rl   g      ?F)r   g?rY   )r   gffffff?)r   r   )tempfileTemporaryDirectoryr   )r   dr   r   r   test_check_patience   s*   


"r   )FFFr   r   )r   Nri   )rp   )ri   T)'rR   r   r^   typingr   r   r   r   numpyr}   r,   logurur   r   	df.configr   r	   df.modelr
   df.utilsr   libdfr   r   r   rQ   boolr`   r/   optim	Optimizerr.   rv   r   r   r   r   r   r   r   r   r   r   <module>   s    

B
%
