o
    9wiA                     @   s   d Z ddlZddlmZ ddlmZ ddlmZmZ ddl	Z	ddl
m  mZ ddlmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZ eeZ dd Z!G dd de"Z#dS )zMain training loop.    N)get_xp)write_and_rename)LogProgressbold   )augmentdistribstates
pretrained)apply_model)ModelEMA)evaluatenew_sdr)svd_penalty)pull_metricEMAc                 C   s   d dd |  D S )N | c                 s   s&    | ]\}}|   d | V  qdS )=N)
capitalize).0keyval r   J/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/demucs/solver.py	<genexpr>   s   $ z_summary.<locals>.<genexpr>)joinitems)metricsr   r   r   _summary   s   r   c                   @   sZ   e Zd Zdd Zdd Zdd Zdedefd	d
ZdedefddZdd Z	dddZ
dS )Solverc              	   C   s  || _ || _|| _|| _t| j|j| j| _t	|| _
tt| j j| _g g d| _| j D ]&}t|j|}|dkrB| jnd}|rY|D ]}| j| t| j||d qHq3tjt|jj|jj |jjdg}	|jjrz|	t t g7 }	dD ]}
t|j|
}|jr|	tt|
  di | q|t!j"j#|	 | _t$ }|j%| _%|j%d | _&|j%d | _'t()d	| j&*  d | _+d
| _,|j-| _-| j-j.| _.| /  d S )N)batchepochr    cpu)device)shiftsame)scaleremixcheckpoint.thzbest.thzCheckpoint will be saved to %sFr   )0argsloadersmodel	optimizerr	   get_quantizerquant	quantizerr   wrapdmodelnextiter
parametersr#   emaskeysgetattremaappendr   r   Shiftintdset
samplerater$   
shift_sameflipFlipChannelsFlipSignprobar   torchnn
Sequentialr   foldercheckpoint_file	best_fileloggerdebugresolve
best_statebest_changedlinkhistory_reset)selfr*   r+   r,   r)   kinddecaysr#   decayaugmentsaugkwxpr   r   r   __init__   sH   
zSolver.__init__c           	      C   s  i }| j  |d< | j |d< | j|d< | j|d< | j|d< | j D ]\}}t|D ]\}}| |d| d| < q,q$t	| j
}t|| W d    n1 sTw   Y  | jj}|r|d | d	kr|d | jjkrt	| jd
|d  d }t|| W d    n1 sw   Y  | jrt	| j}t| j | j}| j|d< t|| W d    n1 sw   Y  d| _d S d S )Nstater,   rO   rL   r)   ema__r   r   checkpoint_z.thF)r+   
state_dictr,   rO   rL   r)   r5   r   	enumerater   rG   rC   save
save_everyepochsrF   rM   rH   r	   serialize_model)	rQ   r!   packagerR   r5   kr8   tmpra   r   r   r   
_serializeM   s4   


$

zSolver._serializec           
   	   C   s|  | j  rUtd| j   t| j d}| j|d  | j|d  |d | j	dd< |d | _
| j D ]\}}t|D ]\}}||d| d	|   q@q8dS | jjrntj| jj| jjd
}| j|  dS | jjrd}| jj}|t| jj | }	td|	 t|	d}|d | _
| jjr| jj|d dd n
| jj|d dd | jjr| j|d  dS dS dS )z8Reset state of the solver, potentially using checkpoint.zLoading checkpoint model: r"   rZ   r,   rO   NrL   r[   r\   )namerepor(   zLoading from %sF)strict)rG   existsrI   inforC   loadr+   load_state_dictr,   rO   rL   r5   r   r_   r)   continue_pretrainedr
   	get_modelpretrained_repor^   continue_fromrF   parentstrcontinue_bestcontinue_opt)
rQ   rd   rR   r5   re   r8   r+   rh   rootcfr   r   r   rP   g   s@   


zSolver._resetr   returnc                 C   s   t |d dt |d dd}d|v rt |d d|d< | jdur*t |d d	|d< d
|v r7t |d
 d|d
< d|v rDt |d d|d< d|v rN|d |d< d|v r[t |d d|d< d|v rht |d d|d< |S )z#Formatting for train/valid metrics.lossz.4freco)rz   r{   nsdr.3fNmsz.2fgradbestbnamepenaltyhloss)formatr/   )rQ   r   lossesr   r   r   _format_train   s$   
zSolver._format_trainc                 C   s   i }d|v rt |d d|d< d|v rt |d d|d< | jjD ]&}d| }||v r4t || d||< d| }||v rFt || d||< q |S )zFormatting for test metrics.sdrr}   r|   sdr_nsdr_)r   r+   sources)rQ   r   r   sourcer   r   r   r   _format_test   s   

zSolver._format_testc                 C   s   | j rtd t| j D ]R\}}| |d }ttd|d  dt|  | |d }ttd|d  dt|  d|v r_| |d }|r_ttd	|d  dt|  qd
}tt	| j | j
jD ] }| j  i }td td | ||d< | |d }ttd|d  dt|  td td | j  t  | j|dd}|}d}t| j }i |d< ||d d< | j
jj}| j D ]c\}	}
t|
D ]M\}}|  | j|dd}W d    n1 sw   Y  d|	 d| }||d |< || }|| }|dr#| }| }||k r/|}|j}|}q|d | ||d d< qW d    n	1 sIw   Y  |d | }t| jj d| |g }|drlt|}nt |}||d d< | j
j!j"d
krt#| j
j!}|$d t  t%| jfddi|}W d    n	1 sw   Y  ||d d< | |d }ttd|d  dt|  ||ks| j
j&j'rttd| t|| _(d| _)|d | j
jj* d
k}|| j
jd k}|s|rmtd td | j
jj+r| j(}nt| j }| j
jj,o|}t-| j|% t  t.| |d|d< W d    n	1 s@w   Y  W d    n	1 sPw   Y  | |d }ttd	|d  dt|  | j/| t0j1d
kr| 2| t3d| j45  |r d S qld S )Nz#Replaying metrics from previous runtrainzTrain Summary | Epoch r   r   validzValid Summary | Epoch testzTest Summary | Epoch r   zF----------------------------------------------------------------------zTraining...zCross validation...F)r   mainr[   r\   r|   r   zvalid.r   r   exactTzNew best valid loss %.4fzEvaluating on the test set...)compute_sdrzCheckpoint saved to %s)6rO   rI   rl   r_   r   r   r   r   rangelenr)   rb   r+   r   _run_one_epochevalrC   no_gradr	   
copy_stater^   r   metricr5   r   swap
startswithrZ   updater   rN   maxminsvdr   dictpopr   r<   train_validrL   rM   everyr   r   
swap_stater   push_metricsr   rankrg   rJ   rG   rK   )rQ   r!   r   	formattedr   bvalidr   rZ   r   rR   r5   re   r8   rh   ab
valid_lossmets	best_lossrW   r   should_evalis_lastr   r   r   r   r      s   
"















"
zSolver.trainTc           !      C   s  | j }|r
| jd n| jd }tjdkr|r|j| ddg| }|d|d   }t|}|jr8t||j}t	t
||| j jj|d}t }	t|D ]\}
}|| j}|rd| |}|jdd}n|d d d	f }|d d dd f }|s| j jrt| j|| j jjd	d
}n| |}|rt| jdr| j||}|j|jksJ |j|jfttd| }|jj dkrt!j"||dd}|#|#d	}|}n&|jj dkrt!j$||dd}|#|}|d }|#d	}n	t%d| j j  t&'|j(|}||  |  }d	}| j)d ur| j)* }|j+j,r ||j+j,| 7 }i }||  |  |d< ||d< |set-||. #d	}d	}t/| jj0||D ]\}}}||d| < ||| 7 }qI||  |d< |r|j1j2d	krt3|j1}|4d t5| jfi |}||d< ||j1j2| 7 }||d< t| jj0D ]\}}|| |d| < q|r|6  d	}g }| j7 D ]}|j8d ur||j8j9: d 7 }|;|j8j9 q|d |d< |jj<rt&j=j>?| j7 |jj< | j j@dkr| jA D ]\}}|j8d u rtBd| q| jCD  | jCE  | jFd D ]}|G  q|	|}| H|} |jGd i |  ~~~~|j|
kr; n| j jIrE|rE n
| j j@dkrN nqK|r_| jFd D ]}|G  qWtJ||
d S )!Nr   r   r   ValidTrainz	 | Epoch )totalupdatesrh   )dimr   )splitoverlaptransform_target   l1none)	reductionmseg      ?zInvalid loss r{   r~   r   r|   r   rz   reco_r   unszno gradr    rJ   r!   r   )Kr)   r*   r   
world_sizesampler	set_epochr   max_batchesr   r   rI   misc
num_printsr   r_   tor#   r   sumvalid_applyr   r+   r   r   r1   hasattrr   shapetupler   r   optimrz   Fl1_lossmeanmse_loss
ValueErrorrC   tensorweightsr/   
model_sizer.   diffqr   detachzipr   r   r   r   r   r   backwardr4   r   datanormr9   	clip_gradrD   utilsclip_grad_norm_flagnamed_parametersprintr,   step	zero_gradr5   r   r   rJ   average)!rQ   r!   r   r)   data_loaderlabelrh   r   logprogaverageridxr   mixestimatedimsrz   r{   r   r~   r   nsdrsr   r|   wrW   r   re   	grad_normgradspnr8   logsr   r   r   r   #  s   












zSolver._run_one_epochN)T)__name__
__module____qualname__rY   rg   rP   r   r   r   r   r   r   r   r   r   r      s    .wr   )$__doc__loggingdorar   
dora.utilsr   dora.logr   r   rC   torch.nn.functionalrD   
functionalr    r   r   r	   r
   applyr   r8   r   r   r   r   r   r   r   r   	getLoggerr   rI   r   objectr   r   r   r   r   <module>   s    
