o
    o°×i\  ã                   @   sÆ  U d dl Z d dlZd dlZd dlZd dlZd dlmZmZmZm	Z	 d dl
Zd dlZd dlZd dlmZ d dlmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZmZ d d
l m!Z!m"Z" d dl#m$Z$m%Z%m&Z& d dl'm(Z(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8 d dl9m:Z: d dl;m<Z= da>da?da@daAee: eBd< eejC eBd< dZDejEdd„ ƒZF		dCdejCdeGde=deHdejIde)deHd eejJ d!eejJ d"eKfd#d$„ZLd"e)fd%d&„ZM	dDd'eeH dejCd(eNd)eNd"ejIf
d*d+„ZOd,eGd"ejJfd-d.„ZPd,eGd"eejJ fd/d0„ZQe R¡ 	1	dEd2ed3ed4ed5ed6edeHd7eeG fd8d9„ƒZSd:d;„ ZTd<d=„ ZUd>d?„ ZVeWd@krad dAlXmYZYmZZZ dBeY_[eZƒ  eFƒ  dS dS )Fé    N)ÚDictÚListÚOptionalÚTuple)Úlogger)ÚTensorÚnnÚoptim)Úset_detect_anomaly)Úset_grad_enabled)Úclip_grad_norm_)ÚNumber)Úcheck_patienceÚ
load_modelÚread_cpÚwrite_cp)ÚCsvÚconfig)Úinit_loggerÚlog_metricsÚlog_model_summary)ÚIstftÚLoss)Úcosine_scheduler)ÚModelParams)Ú
get_device)Ú
as_complexÚas_realÚcheck_finite_moduleÚcheck_manual_seedÚdetach_hiddenÚget_hostÚget_norm_alphaÚmake_np)ÚDF)ÚPytorchDataLoaderFÚstateÚistfté2   c            %      C   s 	  t  ¡ } | jdtdd | jdtdd | jdtdd | jdd	td d
d | jdddd | jdtd dd | jddd | jdddd |  ¡ }tj |j¡sYt	d 
|j¡ƒ‚tj |j¡shtd 
|j¡ƒ tj|jdd tj |jd¡}tj|dd |ja|jd ur˜tr”|j ¡ dkr”tdƒ‚|j}ntrœdnd}ttj |jd¡||jd  tj |jd!¡}t |¡ td"d#td$d%}t|ƒ t d& 
tƒ ¡¡ |jd ur6zDtj tj  tj  tj  tj !t"¡¡¡¡¡ d'd(l#m$} t%ƒ d) tj&d*d$d% }|d)tj&d+d,d% 7 }|||j|d- tj|dd. W n t'y5 } zt (d/t|ƒ› ¡ W Y d }~nd }~ww t) )t)j*t+|jƒ¡ t,ƒ }	t-|	j.|	j/|	j0|	j1|	j2d0a3tj |jd1¡}
tj|
dd td2d3t4d$d%}td4d3t4d$d%}td5d3t4d$d6}t5|j6r|
nd t3d3||d7\}}td8d9td$d%}td:d'td$d%}|d'kr |n|}td;d3t4d$d%}td<d3t4d$d3d=a7t8d–i d>|j“d?|j“d@|	j.“dA|“dB|“dCtdDdEtd$d%“dFtƒ j9dGk“dHtdIdJt:d$d%“d+|	j/“dK|	j0“dL|	j1“dM|	j;“dNt<d3dO“dPtdPdQt:dRd%“dStdTdUt:dRd%“dVtdVdUt:dRd%“dWtdWdUt:dRd%“dXtdXdUt:dRd%“dYtdYdUt:dRd%“dZtd[d\td$d%“d]|“d^|“d_|	j2“d`t7“datdbdct:d$d%“ddtdeg df¢t=tƒd$d%“dgtdhg di¢t=tƒd$d%“dj|“Ž}tdkg t=tƒd$d%}|}|}t>|ƒd'krœdldm„ dndo„ |D ƒD ƒ}|d' d' d'ks—J ‚t dp¡ tdqdrtd$d%}|d'ks«J ‚t?|j6r²|
nd |||ƒ}t@t>|ƒƒ}tAt>|ƒƒ}|j6sÞtj tj |
ds¡¡rÞt Btj |
ds¡¡ z	tC||jdt W n t'y } zt Ddu|› ¡ W Y d }~nd }~ww |rtEjF G|¡}g }tdvdwd$d%}tdxdyd$d%}| Hdzdy¡ Hd{d|¡}td}d~td$d%}tIƒ }tdd3t4d$d6rbtJ||d9 |d€|||d} dw| i}!|! Kd‚dƒ„ | L¡ D ƒ¡ tMd„|d9 › d…|!ƒ | N¡  t Otj |jd!¡¡ tP||ƒD ]â}t>|ƒd'kr²|D ]\}}"||krtQ|"|ƒ}q||kr²t d†|› ¡ | R|d$¡ t@t>|ƒƒ}tAt>|ƒƒ}|}tJ|||d$|||||d‡	}#dw|#i}!z|jSd' dˆ |!dˆ< W n
 tTyØ   Y nw trè|! Kd‰dƒ„ | L¡ D ƒ¡ tMd„|› dŠ|!ƒ tU|d*|
|d9 ƒ tU|d‹|
|d9 ƒ | N¡  tJ|||d€|||d} dw| i}!|! KdŒdƒ„ | L¡ D ƒ¡ |!| }tU|d*|
|d9 ||d tMd„|› d…|!ƒ tV|
|||d3dŽsG ntWrSt d¡ tXd'ƒ | N¡  qvt5|
t3|d||d‘\}}tJ|||d’|||d}$dw|$i}!|! Kd“dƒ„ | L¡ D ƒ¡ tMd„|› d”|!ƒ t d•¡ d S )—NÚdata_config_filezPath to a dataset config file.)ÚtypeÚhelpÚdata_dirz5Path to the dataset directory containing .hdf5 files.Úbase_dirz5Directory to store logs, summaries, checkpoints, etc.z--host-batchsize-configz-bz*Path to a host specific batch size config.)r*   Údefaultr+   z--no-resumeÚstore_falseÚresume)ÚactionÚdestz--log-levelzALogger verbosity. Can be one of (trace, debug, info, error, none)z--debugÚ
store_true)r1   z
--no-debugÚdebugzDataset config not found at {}zData directory not found at {}T©Úexist_okÚ	summariesz*Either specify debug or a manual log levelÚDEBUGÚINFOz	train.log)ÚfileÚlevelÚmodelz
config.iniÚSEEDé*   Útrain©ÚsectionzRunning on device {}r   )ÚmainÚ_r<   Úfft_sizeÚdf)Úhost_key)Úallow_reloadz1Could not apply host specific batch size config: )ÚsrrD   Úhop_sizeÚnb_bandsÚmin_nb_erb_freqsÚcheckpointsÚ	MASK_ONLYFÚDF_ONLYÚJIT)ÚcastrA   )ÚjitÚ	mask_onlyÚtrain_df_onlyÚ
BATCH_SIZEé   ÚBATCH_SIZE_EVALÚOVERFITÚLOG_TIMINGS©rA   ÚsaveÚds_dirÚ	ds_configrH   Ú
batch_sizeÚbatch_size_evalÚnum_workersÚNUM_WORKERSé   Ú
pin_memoryÚcudaÚ	max_len_sÚMAX_SAMPLE_LEN_Sg      @rI   Únb_erbÚnb_specÚ
norm_alpha©ÚlogÚp_reverbgš™™™™™É?Ú
distortionÚp_bw_extÚp_bandwidth_extç        Ú
p_clippingÚ	p_zeroingÚp_air_absorptionÚp_interfer_spÚprefetchÚNUM_PREFETCH_BATCHESé    ÚoverfitÚseedrK   Úlog_timingsÚglobal_sampling_factorÚGLOBAL_DS_SAMPLING_Fç      ð?ÚsnrsÚDATALOADER_SNRS)éûÿÿÿr   é   é
   é   é(   ÚgainsÚDATALOADER_GAINS)iúÿÿÿr   é   Ú	log_levelÚBATCH_SIZE_SCHEDULINGc                 S   s$   g | ]}t |d  ƒt |d ƒf‘qS )r   rU   )Úint©Ú.0Úbs© r   úF/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/df/train.pyÚ
<listcomp>´   s    ÿzmain.<locals>.<listcomp>c                 s   s    | ]}|  d ¡V  qdS )ú/N)ÚsplitrŠ   r   r   rŽ   Ú	<genexpr>µ   s   € zmain.<locals>.<genexpr>z%Running with learning rate schedulingÚ
MAX_EPOCHSr   z	.patience)ÚverbosezFailed to print model summary: ÚVALIDATION_CRITERIAÚlossÚVALIDATION_CRITERIA_RULEÚminÚlessÚmoreÚmaxÚEARLY_STOPPING_PATIENCEr€   Ú
START_EVALÚvalid)r<   ÚepochÚloaderr‘   ÚoptÚlossesÚsummary_dirc                 S   ó&   i | ]\}}|t  t  |¡¡ ¡ “qS r   ©ÚtorchÚmeanÚstackÚitem©r‹   ÚnÚvalsr   r   rŽ   Ú
<dictcomp>ã   ó   & zmain.<locals>.<dictcomp>ú[z	] [valid]z)Batch scheduling | Setting batch size to )	r<   rŸ   r    r‘   r¡   r¢   r£   Úlr_scheduler_valuesÚwd_scheduler_valuesÚlrc                 S   r¤   r   r¥   rª   r   r   rŽ   r­   	  r®   z	] [train]r¡   c                 S   r¤   r   r¥   rª   r   r   rŽ   r­     r®   )ÚmetricÚcmp)Úmax_patienceÚ
new_metricr´   Úraise_z Stopping training due to timeoutÚbest)rQ   rŸ   rR   rS   Útestc                 S   r¤   r   r¥   rª   r   r   rŽ   r­   ?  r®   z] [test]zFinished trainingr   )YÚargparseÚArgumentParserÚadd_argumentÚstrÚ
parse_argsÚosÚpathÚisfiler)   ÚFileNotFoundErrorÚformatÚisdirr,   ÚNotADirectoryErrorÚmakedirsr-   Újoinr4   r‡   ÚlowerÚ
ValueErrorr   r   Úloadr‰   r   r   Úinfor   Úhost_batchsize_configÚsysÚappendÚdirnameÚabspathÚ__file__Úscripts.set_batch_sizerB   r!   ÚgetÚ	ExceptionÚerrorÚsignalÚSIGUSR1Úget_sigusr1_handlerr   r$   rH   rD   rI   rf   Úmin_nb_freqsr&   Úboolr   r0   ry   Ú
DataLoaderr*   ÚfloatÚnb_dfr"   r   ÚlenÚload_optÚ	setup_lrsÚ	setup_wdsÚremover   Úwarningr¦   rQ   ÚscriptÚreplaceÚsetup_lossesÚ	run_epochÚupdateÚget_summariesr   Úreset_summariesrZ   Úranger˜   Úset_batch_sizeÚparam_groupsÚAttributeErrorr   r   Úshould_stopÚexit)%ÚparserÚargsr£   r‡   Úconfig_filerx   rì   ÚkeyÚeÚpÚcheckpoint_dirrR   rS   rQ   r<   rŸ   rŒ   Úbs_evalrw   Ú
dataloaderÚbatch_size_schedulingÚscheduling_bsÚprev_scheduling_bsÚ
max_epochsr¡   ÚlrsÚwdsÚval_criteriaÚval_criteria_typeÚval_criteria_ruleÚpatiencer¢   Úval_lossÚmetricsÚbÚ
train_lossÚ	test_lossr   r   rŽ   rB   .   s2  ÿÿûü

"ÿ €ÿû
ûÿþýüûúùø	÷
öõô
óòñðïîíìëêéèçæåä%ÿ
ü €ÿù	ÿ

€
÷ÿÿù	ÿÿû

úù	rB   r<   rŸ   r    r‘   r¡   r¢   r£   r°   r±   Úreturnc	           (         sî  t dtddd}	| |¡‰ t d ||ˆ ¡¡ t ddtdd}
|
r&t d	¡ tƒ }g }|dk}| j|d
 t	p9| |_
| |¡d }|rF|nd}d}|| |¡ }t| ||¡ƒD ]i\}}| ¡  || }|d usn|d ur”|jD ]"}|d urƒ|| | dd¡ |d< |d ur“|d dkr“|| |d< qq|jd us›J ‚|jd us¢J ‚|jj|dd}t|jj|ddƒ}|jj|dd}|jj|dd}|jj|dd}t|
oÐ|ƒF t|ƒ0 |sãt|ƒ ¡ }nt|ƒ}| j|||d\}}}}z|j||||||d} W nL tyL }! z?dt|!ƒ ¡ v sdt|!ƒ ¡ v rFt d t|!ƒ¡¡ t | ƒ |d7 }|t!kr4|!‚W Y d }!~!W d   ƒ W d   ƒ qY|!‚d }!~!ww |rþz|  "¡  t#|  $¡ ddd W nš t%yù }! zt|!ƒ}"d|" ¡ v sxd|"v rót | ƒ t &|"¡ t'j(t'j) *|d¡dd t+|j,d ƒD ]6}#|j-|#  .¡ }$t/| 0¡ | 0¡ | 0¡ |j 0¡ | 0¡  1¡ t'j) *|d¡|d|› d|› d|#› d|$›  |#d  q”t2| |||||||ƒ |d7 }|t!krá|!‚W Y d }!~!W d   ƒ W d   ƒ qY|!‚d }!~!ww | 3¡  t4| ƒ W d   ƒ n	1 sw   Y  W d   ƒ n	1 sw   Y  | 5|  0¡ ¡ ||	 dkrÃt6 7|d!d … ¡ 8¡  9¡ }%t6 :|%¡rGt | ƒ d"|% .¡ i}&|d ur[|jd d |&d< |d uri|jd d |&d#< t;r€|j<d d$…  =¡ |&d%< |j<d$  8¡ |&d&< t	r‘|& >‡ fd'd(„| ?¡ D ƒ¡ t|ƒ @tt|ƒƒ¡}'tAd)|› d*|'› d+|› d,|&ƒ t/| 0¡ | 0¡ | 0¡ |j 0¡ | 0¡  1¡ ||d- qYzt2| |||||||ƒ W n tByë }  zt &t| ƒ¡ W Y d } ~ nd } ~ ww t6 7|¡ 8¡  9¡  .¡ S ).NÚLOG_FREQéd   r?   )rP   r.   rA   z$Start {} epoch {} with batch size {}ÚDETECT_ANOMALYFr@   zRunning with autograd profiling)ÚmoderU   r>   r   Úlr_scaler²   Úweight_decayT)Únon_blocking)ÚspecÚfeat_erbÚ	feat_spec)r}   ÚnanÚfinitez/NaN in loss computation: {}. Skipping backward.r|   )Úerror_if_nonfinitez
non-finiter5   Ú_eÚ_iÚ_bÚ_ds)ÚprefixÚidxiœÿÿÿr–   ÚwdéÿÿÿÿÚt_sampleÚt_batchc              
      s0   i | ]\}}|t  t  |ˆ  d … ¡¡ ¡ “qS ©Nr¥   rª   ©rŒ   r   rŽ   r­   µ  s    ÿÿzrun_epoch.<locals>.<dictcomp>r¯   z] [r   ú])r  )Cr   r‰   Úget_batch_sizer   rË   rÃ   rÚ   r   r?   r4   Ústore_lossesrÞ   Ú	enumerateÚ
iter_epochÚ	zero_gradrí   rÓ   r  r  Útor   ÚnoisyÚspeechÚsnrr
   r   ÚcloneÚforwardrÔ   r½   rÈ   rã   r   ÚMAX_NANSÚbackwardr   Ú
parametersÚRuntimeErrorrÕ   r¿   rÆ   rÀ   rÇ   rë   ÚshapeÚidsr©   Úsummary_writeÚdetachrÜ   ÚcleanupÚstepr    rÎ   r¦   r¨   r§   ÚcpuÚisnanry   ÚtimingsÚsumrè   ré   Úrjustr   ÚUnboundLocalError)(r<   rŸ   r    r‘   r¡   r¢   r£   r°   r±   Úlog_freqÚdetect_anomalyÚdevÚl_memÚis_trainÚ	max_stepsrx   Ún_nansÚstart_stepsÚiÚbatchÚitÚparam_groupr  r  r*  Úcleanr}   ÚinputÚenhÚmÚlsnrÚotherÚerrrõ   Úe_strÚ	batch_idxÚ	clean_idxÚl_meanÚl_dictr8  r   r"  rŽ   rç   D  s  


€ý$

í€ø	


ø


Ñ1€è
Í€ 4


þÿ
ù€	€ÿrç   c                  C   sN   t d usJ ‚tƒ } t| j| jt t  ¡  ¡ ¡ƒ 	t
ƒ ¡att tƒ 	t
ƒ ¡}|S r!  )r&   r   r   rD   rI   r¦   Ú	as_tensorÚ
fft_windowÚcopyr)  r   r'   r   )rö   r–   r   r   rŽ   ræ   Ì  s   ÿræ   Úcp_dirrR   Údf_onlyc              
      s   t ddtdd‰t ddtdd‰t ddtdd‰t d	d
tdd ¡ }t dddgttƒddd‰ |rJg }| ¡ D ]\}}d|v sHd|v sH| |¡ q7n|rVdd„ | ¡ D ƒ}n| ¡ }‡ ‡‡fdd„‡ ‡‡fdd„‡‡‡fdd„‡‡‡fdd„dœ}||vr‹td|› dt	| 
¡ ƒ› ƒ‚|| |ƒ}	t d|	› ¡ | d urÂz
t|	d| dd W n tyÁ }
 zt d|
› ¡ W Y d }
~
nd }
~
ww |	jD ]}| d ˆ¡ qÅ|	S )!NÚLRgü©ñÒMb@?r	   r@   Úmomentumr   r  çš™™™™™©?Ú	optimizerÚadamwÚ	opt_betasgÍÌÌÌÌÌì?g+‡ÙÎ÷ï?FrY   ÚdfrnnÚdf_decc                 s   s$    | ]\}}d |  ¡ v r|V  qdS )rE   N)rÈ   )r‹   r«   rö   r   r   rŽ   r’   ê  s   €" zload_opt.<locals>.<genexpr>c                    ó   t j| ˆˆˆ ddS ©NT)r²   r  ÚbetasÚamsgrad)r	   ÚAdam©rö   ©rf  Údecayr²   r   rŽ   Ú<lambda>î  ó    zload_opt.<locals>.<lambda>c                    rd  re  )r	   ÚAdamWri  rj  r   rŽ   rl  ï  rm  c                    s   t j| ˆˆdˆ dS )NT)r²   r]  Únesterovr  )r	   ÚSGDri  ©rk  r²   r]  r   rŽ   rl  ð  rm  c                    s   t j| ˆˆˆ dS )N)r²   r]  r  )r	   ÚRMSpropri  rq  r   rŽ   rl  ñ  s    )Úadamr`  ÚsgdÚrmspropzUnsupported optimizer: z. Must be one of zTraining with optimizer r¡   ri   z Could not load optimizer state: Ú
initial_lr)r   rÜ   r½   rÈ   r   Únamed_parametersrÎ   r1  rÉ   ÚlistÚkeysr   r4   r   rÕ   rí   Ú
setdefault)rZ  r<   rR   r[  r_  Úparamsr«   rö   Ú	supportedr¡   rõ   Úgroupr   )rf  rk  r²   r]  rŽ   rß   Ú  sL   ÿ
€þüÿ€ÿ
rß   Ústeps_per_epochc           
      C   s¦   t  dtd¡}t  dtd¡}t ddtdd}t dd	tdd}||k s$J ‚t d
dtdd}t ddtdd}t ddtdd}t ddtdd}t|||| |||||d	}	|	S )Nr²   r	   rý   r?   Úlr_mingíµ ÷Æ°>r@   Ú	lr_warmupg-Cëâ6?Úwarmup_epochsé   Úlr_cycle_mulr|   Úlr_cycle_decayg      à?Úlr_cycle_epochsr  )ÚepochsÚniter_per_epr  Ústart_warmup_valueÚinitial_ep_per_cycleÚcycle_decayÚ	cycle_mul)r   rÓ   rÜ   r‰   r   )
r~  r²   Ú
num_epochsr  r€  r  rƒ  r„  r…  Ú	lr_valuesr   r   rŽ   rà     s*   ÷rà   c                 C   s„   t ddtdd}t ddtdd}|dkrd S |dkr1d}t d	¡ t d
|› d¡ t  dd|¡ t  dtd¡}t||| |d}|S )Nr  r^  r	   r@   Úweight_decay_endr  ro   gê-™—q=z?Got 'weight_decay_end' value > 0, but weight_decay is disabled.z Setting initial weight decay to Ú.rý   r?   )r‡  r†  )r   rÜ   r   rã   Ú	overwriterÓ   r‰   r   )r~  rk  Ú	decay_endrŒ  Údecay_valuesr   r   rŽ   rá     s   
ÿrá   r?   rK  r*  rM  r}   rO  r  c              	   C   s  t d usJ ‚tƒ }|jd }	|d u rt |	¡}||  ¡  ¡  ¡ }
dtdtfdd„}t	 
tj ||› d|
› d¡|| | ƒ|j¡ t	 
tj ||› d|
› d¡||| ƒ|j¡ t	 
tj ||› d	|
› d¡||| ƒ|j¡ tjtj ||› d
|
› d¡||  ¡  ¡  ¡ dd d S )Nr   Úxr	  c                 S   s   t  t tt|  ¡ ƒƒ¡¡S r!  )r¦   rW  r&   Ú	synthesisr#   r   r6  )r“  r   r   rŽ   r”  @  s   z summary_write.<locals>.synthesisÚ
_clean_snrz.wavÚ
_noisy_snrÚ_enh_snrÚ	_lsnr_snrz.txtz%.3f)Úfmt)r&   r   r3  ÚrandomÚ	randranger6  r9  r©   r   Ú
torchaudiorZ   r¿   rÀ   rÇ   rH   ÚnpÚsavetxtÚnumpy)rK  r*  rM  r}   rO  r£   r  r  rö   rŒ   r,  r”  r   r   rŽ   r5  ,  s*   

&ÿ&ÿ&ÿ
ýr5  c                  O   s   d S r!  r   )Ú__argsÚ__kwargsr   r   rŽ   Úsummary_noopS  s   r¢  c                    s   ‡ fdd„}|S )Nc                     s>   t  d¡ datj ˆ d¡}t  d|› ¡ t|dƒ ¡  d S )Nz5Received timeout signal. Stopping after current epochTÚcontinuezWriting Úw)r   rã   rï   r¿   rÀ   rÇ   ÚopenÚclose)r   Úcontinue_file©r-   r   rŽ   ÚhX  s
   
zget_sigusr1_handler.<locals>.hr   )r-   r©  r   r¨  rŽ   rØ   W  s   rØ   c                  G   s*   dd l }| D ]}~q| ¡  tj ¡  d S )Nr   )ÚgcÚcollectr¦   rc   Úempty_cache)rò   rª  Úargr   r   rŽ   r7  c  s
   r7  Ú__main__)ÚicÚinstallT)NN)FF)r?   N)\rº   r¿   rš  rÖ   rÍ   Útypingr   r   r   r   rŸ  r  r¦   rœ  Úlogurur   r   r   r	   Útorch.autograd.anomaly_moder
   Útorch.autograd.grad_moder   Útorch.nn.utils.clip_gradr   Útorch.typesr   Údf.checkpointr   r   r   r   Ú	df.configr   r   Ú	df.loggerr   r   r   Údf.lossr   r   Údf.lrr   Údf.modelr   Ú
df.modulesr   Údf.utilsr   r   r   r   r    r!   r"   r#   Úlibdfr$   Ú	libdfdatar%   rÛ   rï   r4   ry   r&   Ú__annotations__ÚModuler/  ÚcatchrB   r‰   r½   Ú	OptimizerÚndarrayrÜ   rç   ræ   rÚ   rß   rà   rá   Úno_gradr5  r¢  rØ   r7  Ú__name__Úicecreamr¯  r°  ÚincludeContextr   r   r   rŽ   Ú<module>   sÊ   
 (

  ÷ÿþýüûúùø	÷

ö 	ÿÿÿÿÿ
þ)øÿþýüûúø&
	
û