o
    itL                     @   s,  d Z ddlZddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlm  m  mZ ddlm  m  mZ ddlmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z- dd Z.dd Z/dS )z=Training/decoding definition for the speech recognition task.    N)training)TransformDataset)
extensions)CompareValueTriggeradadelta_eps_decayadd_results_to_jsonchainer_loadget_model_confrestore_snapshot)ASRInterface)set_deterministic_chainer)dynamic_import)LoadInputsAndTargets)make_batchset)BaseEvaluator)ShufflingEnabler'ToggleableShufflingMultiprocessIterator!ToggleableShufflingSerialIterator)TensorboardLogger)check_early_stopset_early_stopc           %         s0
  t dtj  t  tjjst d tjjst d t	 j
d}t|d }W d   n1 s6w   Y  t| }t||d  d d d	 d
 }t||d  d d d	 d
 }t dt|  t dt|   jdkrd}t d n jdkrd}t d nd}t d t d j  t j}||| dd}t|tsJ | }	tj jst j  jd }
t	|
d#}t d|
  |tj||t fddddd W d   n1 sw   Y  t t  D ]}t d | d! tt |   q j!d
kr.d}tj"|#  |$  t d" n9d
kr`d}d#|i}t%j&'d
D ]	|d$ < q@t d%  t d& j( j( j! f  nd'}t d(  j)d)krvtj*j+ j,d*}n% j)d+krtj*- }n j)d,krtj*j-dd-d.d/d0}nt.d1/ j)|0| |1tj23 j4 |j5|j6d d2}t	 j7d}t|d }W d   n	1 sw   Y  t	 j
d}t|d }W d   n	1 sw   Y  t8d3d j9d4did5t8d3d j9d4did5} j:d'kp j:dk j;}d
krlt<| j( j= j> j? j!d
kr+ j!nd
 j@ jA jB jC jDddd6} jEdkrStFtG|d
 jEd7d8 d9g}ntHtG|d
 d:g}|jI|d ||||d;}n j@d<vr| j(dkr|t.d=g t%j&'D ] fd>d?tJ|K D }t<| j( j= j> j?g7 qtLd@dA D }D ]}|tM|krt%j&'|tM| D ]
}||| g7 }qÐq jEdkr fdBdAt%j&'D }nfdCdAt%j&'D }|jN||||dD}tOjP| jQdEf jdF}r#|jRtS| j:d'kr j:n jQdEfdG  j)d,kr?ddHlTmU} |jR|dI jV jW jXdJdKdG  jYrKtjZ[ jY| t<| j( j= j> j? j!d
kr^ j!nd
 j@ jA jB jC jDdddL} jEdkrtj\j]tG||d
dd jEd7d8dM}ntj\j^tG||d
dddN}|Rt_||||dO  j`dkr jdkrt t|K d j` dPdQ ddR}ta|dSr|jbjc}|jbjd} n|jc}|jd} t dT | || jdU ||||	dV}!|jR|!dWdG nd}!|jRtejfdXdYdWdG |Rtejgg dZdEd[d\ |Rtejgd]d^gdEd_d\ |jRteh|d`tOjijdadG |dkr6|jRteh|dbtOjikd^dG  j)d)kr jldckri|dkri|jRtm| jdd tnd^dedQ dG |jRto jptnd^dfdQ dG n' jldgkr|jRtm| jdh tndadidQ dG |jRto jptndadjdQ dG |Rtejq jrdkfdG g dl}" j)d)kr|jRtesdmdndQ  jrdkfdG |"tdm |jRteu|" jrdkfdG |Rtejv jrdo tw|   jxdur jxdpkrzddqlymz}# W n t{y   t |dr  w |# jx}$|jRt}|$|! jrdkfdG |~  t| jQ dS )sz\Train with the given args.

    Args:
        args (namespace): The program arguments.

    chainer version = zcuda is not availablezcudnn is not availablerbuttsNr   inputshape   outputz#input dims : z#output dims: g      ?ctczPure CTC modeg        attzPure attention modemtlzMultitask learning modezimport model module: F)flag_returnz/model.jsonwbzwriting a model config file to    Tindentensure_ascii	sort_keysutf_8ARGS: : zsingle gpu calculation.mainzsub_%dz#multi gpu calculation (#gpus = %d).z0batch size is automatically increased (%d -> %d)zcpu calculationadadelta)epsadamnoamg?g\(\?g&.>)alphabeta1beta2r.   zargs.opt={})subsampling_factorasrtrain)modeload_outputpreprocess_confpreprocess_args)	min_batch_sizeshortest_firstcount
batch_binsbatch_frames_inbatch_frames_outbatch_frames_inoutiaxisoaxis      
batch_sizen_processes
n_prefetchmaxtasksperchildshufflerG   rK   )	converterdevice
accum_grad)autoseqzH--batch-count 'bin' and 'frame' are not implemented in chainer multi gpuc                    s&   i | ]\}\}}|  kr||qS  rR   ).0ikv)gidngpurR   R/home/ubuntu/.local/lib/python3.10/site-packages/espnet/asr/chainer_backend/asr.py
<dictcomp>   s     ztrain.<locals>.<dictcomp>c                 S   s   g | ]}t |qS rR   )len)rS   train_subsetrR   rR   rY   
<listcomp>   s    ztrain.<locals>.<listcomp>c              
      s.   g | ]}t t| d  jdd dqS )r   rD   rE   rF   )r   r   n_iter_processesrS   rW   )argsload_trtrain_subsetsuse_sortagradrR   rY   r]      s    	c                    s&   g | ]}t t|  d  dqS )r   rL   )r   r   r_   )ra   rb   rc   rR   rY   r]     s    )rM   devicesepoch)out)trigger)VaswaniRuler1   )dwarmup_stepsscale)r   	iteration)r;   r=   r>   r?   r@   rA   rB   rC   )rG   repeatrK   rH   rI   rJ   )rG   rm   rK   )rM   rN   c                 S   s   t | d d d d d S )Nr   r   r   r   )int)xrR   rR   rY   <lambda>O  s    ztrain.<locals>.<lambda>)keyreversemodulez Using custom PlotAttentionReportz/att_ws)rM   	transformrN   r4   )r   re   zsnapshot.ep.{.updater.epoch})filename)	main/lossvalidation/main/lossmain/loss_ctcvalidation/main/loss_ctcmain/loss_attvalidation/main/loss_attzloss.png)	file_namemain/accvalidation/main/acczacc.pngzmodel.loss.bestrw   zmodel.acc.bestaccz/model.acc.bestc                 S      | |kS NrR   
best_valuecurrent_valuerR   rR   rY   rp         c                 S   r   r   rR   r   rR   rR   rY   rp     r   lossz/model.loss.bestc                 S      | |k S r   rR   r   rR   rR   rY   rp     r   c                 S   r   r   rR   r   rR   rR   rY   rp     r   rl   )re   rl   rv   rx   rz   rw   ry   r{   r}   r~   elapsed_timer.   c                 S   s   | j djS )Nr+   )updaterget_optimizerr.   )trainerrR   rR   rY   rp     s    )update_interval )SummaryWriterzPlease install tensorboardx)logginginfochainer__version__r   cuda	availablewarningcudnn_enabledopen
valid_jsonjsonloadlistkeysrn   strmtlalphamodel_moduler   
isinstancer   get_total_subsampling_factorospathexistsoutdirmakedirswritedumpsvarsencodesortedrX   get_device_from_iduseto_gpusixmovesxrangerG   opt
optimizersAdaDeltar.   AdamNotImplementedErrorformatsetupadd_hook	optimizerGradientClipping	grad_clipcustom_converter	subsample
train_jsonr   r9   	sortagradrO   r   	maxlen_in
maxlen_outminibatchesbatch_countr>   r?   r@   rA   r^   r   r   r   custom_updater	enumerateitemsmaxr[   custom_parallel_updaterr   Trainerepochsextendr   0espnet.nets.chainer_backend.transformer.trainingrh   adimtransformer_warmup_stepstransformer_lrresumeserializersload_npz	iteratorsMultiprocessIteratorSerialIteratorr   num_save_attentionhasattrrs   calculate_all_attentionsattention_plot_classr   snapshot
PlotReportsnapshot_objecttriggersMinValueTriggerMaxValueTrigger	criterionr
   r   r   	eps_decay	LogReportreport_interval_itersobserve_valueappendPrintReportProgressBarr   tensorboard_dirtensorboardXr   	Exceptionerrorr   runr   )%r`   fr   r   idimodimmtl_modemodel_classmodeltotal_subsampling_factor
model_confrq   gpu_idrd   r   rM   r   load_cvrO   r6   train_itersr   train_json_subsetmaxlenr\   rT   r   rh   valid
valid_iterdata
att_vis_fn
plot_classatt_reporterreport_keysr   writerrR   )r`   rW   ra   rX   rb   rc   rY   r6   -   s  

  




$




	


	





	






	



r6   c              	   C   s  t dtj  t|  t| j| j\}}}tt	| 
 D ]}t d| d tt	| |   qt d| j  t|drC|j}nd}t|}||||}t|tsVJ t| j| | jr|t| j| j}ttt|j|j|j}	t| j|	 nd}	| jrt| j| j}|j}
dd	 t|jD }ttt|
|j|j}t| j| |	durtt |j!|	j!|
|}	ntt"|j!|
|}	t#| j$d
}t%&|d }W d   n1 sw   Y  t'ddd| j(du r|j(n| j(ddid}i }t) E t|
 dD ]5\}}t d| |t|
  ||| fg}||d d }|*|| |j|	}t+|| ||j||< qW d   n	1 sFw   Y  t#| j,d}|-t%j.d|idddd/d W d   dS 1 sow   Y  dS )z]Decode with the given args.

    Args:
        args (namespace): The program arguments.

    r   r)   r*   zreading model parameters from r   z'espnet.nets.chainer_backend.e2e_asr:E2ENc                 S   s   i | ]\}}||qS rR   rR   )rS   rT   ro   rR   rR   rY   rZ     s    zrecog.<locals>.<dictcomp>r   r   r5   Fr6   )r7   r8   sort_in_input_lengthr9   r:   r   z(%d/%d) decoding r   r"   r#   Tr$   r(   )0r   r   r   r   r   r	   r   r   r   r   r   r   r   r   r   r   r   r   rnnlm
rnnlm_conf
lm_chainerClassifierWithStateRNNLMr[   	char_listlayerunit
word_rnnlmword_rnnlm_confchar_list_dictr   extlm_chainerMultiLevelLM	predictorLookAheadWordLMr   
recog_jsonr   r   r   r9   no_backprop_mode	recognizer   result_labelr   r   r   )r`   r   r   
train_argsrq   r   r   r   
rnnlm_argsr  	word_dict	char_dictr  r   jsload_inputs_and_targetsnew_jsidxnamebatchfeat
nbest_hypsrR   rR   rY   recog  s   $



$r#  )0__doc__r   r   r   r   r   r   chainer.datasetsr   chainer.trainingr   espnet.lm.chainer_backend.extlmlmchainer_backendextlmr  espnet.lm.chainer_backend.lmr  espnet.asr.asr_utilsr   r   r   r   r	   r
   espnet.nets.asr_interfacer    espnet.utils.deterministic_utilsr   espnet.utils.dynamic_importr   espnet.utils.io_utilsr   espnet.utils.training.batchfyr   espnet.utils.training.evaluatorr   espnet.utils.training.iteratorsr   r   r   (espnet.utils.training.tensorboard_loggerr   !espnet.utils.training.train_utilsr   r   r6   r#  rR   rR   rR   rY   <module>   s4       5