o
    i:                     @   sr  d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z% ej&' rwddl(m)Z) eej*edkrddl+m,Z,m-Z- n	edddZ-dZ,zddl.Z.W n e/y   dZ.Y nw ej0G dd de!Z1G dd de Z2dS )z&Trainer module for GAN-based training.    N)contextmanager)DictIterableListOptionalSequenceTuple)parse)check_argument_types)AbsBatchStepSchedulerAbsScheduler)	to_device)recursive_average)DistributedOption)SubReporter)TrainerTrainerOptions)build_dataclass)str2bool)ReduceOpz1.6.0)
GradScalerautocastTc                 c   s    d V  d S N )enabledr   r   M/home/ubuntu/.local/lib/python3.10/site-packages/espnet2/train/gan_trainer.pyr   !   s   
r   c                   @   s   e Zd ZU dZeed< dS )GANTrainerOptionsz(Trainer option dataclass for GANTrainer.generator_firstN)__name__
__module____qualname____doc__bool__annotations__r   r   r   r   r   -   s   
 r   c                   @   s   e Zd ZdZedejdefddZedej	fddZ
ed	ejjd
eeee eeejf f  deejj deee  dee dedededefddZee d	ejjd
eeeejf  dedededdfddZ dS )
GANTrainerzTrainer for GAN-based training.

    If you'd like to use this trainer, the model must inherit
    espnet.train.abs_gan_espnet_model.AbsGANESPnetModel.

    argsreturnc                 C   s   t  sJ tt|S )z@Build options consumed by train(), eval(), and plot_attention().)r
   r   r   )clsr%   r   r   r   build_options<   s   

zGANTrainer.build_optionsparserc                 C   s   |j dtddd dS )z)Add additional arguments for GAN-trainer.z--generator_firstFz"Whether to update generator first.)typedefaulthelpN)add_argumentr   )r'   r)   r   r   r   add_argumentsB   s   
zGANTrainer.add_argumentsmodeliterator
optimizers
schedulersscalerreporteroptionsdistributed_optionc
           &         s  t  sJ |j}
|j}|j}|j}|j}|j}|j}|j}|j	}|	j
}|dkr+td|
r1td|du rLztt|d d}W n tyK   d}Y nw |  d}td	|d	kr]d
nd}t }t||ddD ]\}\}}t|tsJ t||rtj
|tj |d	kr |S t||d	krd
nd}|rd}qmt }|rddg}nddg}|D ]}t|du || d~ |d&d|dki|}t|tr6|d }|d }|d } | d  dur5t t!s5t tj"st#dt   $ dkrt#d $  d $ dkr1 D ]}!|! d	 kr(t#dq d	 %  n	 %  nt#dW d   n	1 sEw   Y  dd |& D }|dks[|rq|| |j' ( }t)|| |\}} ||  }|r{|tj
* 9 }W d   n	1 sw   Y  |+||  || d |dur|,|-  n|-  W d   n	1 sw   Y  |durt|D ]\}"}# dur|" krԐq|.|# qd}$|dkrtj/j0j1|2 ||d}$t|$tj"st|$}$|$du st3|$rYd}|| d @ tt4||D ]0\}"\}#}% dur)|" kr)q|dur8|5|# |6  n|#5  t|%t7rF|%5  qW d   n	1 sSw   Y  n-t89d!|$ d" |durt|D ]\}"}# dur{|" kr{qk|5|# |6  qkt|D ]	\}"}#|#:  q|+ fd#dt|  j;D  |+| d$t | i t }q|+d%t | i t }|<  || d	krt8=|>|  |dur|?||  |r|@  qm|r|Ad tj
|tj |S )'zTrain one epoch.   z6accum_grad > 1 is not supported in GAN-based training.z2grad_noise is not supported in GAN-based training.N   
   d   Tr   cudacpu	iter_timeF	generatordiscriminator_forward_timeforward_generatorlossstatsweight	optim_idxz4optim_idx must be int or 1dim torch.Tensor, but got    z
dim tensorz@optim_idx must be 1dim tensor having same values for all entrieszmodel output must be dict.c                 S   s   i | ]\}}|d ur||qS r   r   ).0kvr   r   r   
<dictcomp>   s    z.GANTrainer.train_one_epoch.<locals>.<dictcomp>_backward_timeg        )max_norm	norm_type_optim_step_timezThe grad norm is z. Skipping updating the model.c                    s.   i | ]\}}d |v rd  d| |d  qS )lroptim_lrr   )rG   ipgrE   r   r   rJ     s
    _train_time
train_timer   )Br
   
grad_noise
accum_grad	grad_clipgrad_clip_typelog_intervalno_forward_runngpu	use_wandbr   distributedNotImplementedErrormaxlen	TypeErrortraintorchtensortotimeperf_counter	enumeratemeasure_iter_time
isinstancedictr*   
all_reducer   SUMr   r   measure_timegetintTensorRuntimeErrordimitemitemsdtypesumr   get_world_sizeregisterscalebackwardunscale_nnutilsclip_grad_norm_
parametersisfinitezipstepupdater   loggingwarning	zero_gradparam_groupsnextinfolog_messagetensorboard_add_scalar	wandb_logfill_)&r'   r/   r0   r1   r2   r3   r4   summary_writerr5   r6   rW   rX   rY   rZ   r[   r\   r]   r^   r   r_   all_steps_are_invaliditerator_stop
start_timeiiter_batchturn_start_timeturnsturnretvalrB   rC   rD   rI   iopt	optimizer	grad_norm	schedulerr   rT   r   train_one_epochL   sJ  
 , 


%6








	




zGANTrainer.train_one_epochNc                 C   sZ  t  sJ |j}|j}|j}|j}	|  td|dkr dnd}
|D ]t\}}t	|t
s4J t||rEtj|
tj |
dkrE d
S t||dkrMdnd}|rSq%|	rZddg}nddg}|D ]4}|dd|dki|}t	|t
r{|d }|d }n|\}}}|d	ks|rt|||\}}||| q`|  q%|r|
d	 tj|
tj d
S d
S )zValidate one epoch.r   r;   r<   r>   r?   rA   rC   rD   r7   Nr   )r
   r]   r\   r_   r   evalre   rf   rg   rl   rm   r*   rn   r   ro   r   r   r{   r   r   )r'   r/   r0   r4   r5   r6   r]   r\   r_   r   r   r   r   r   r   r   rC   rD   r   r   r   validate_one_epoch4  sB   






zGANTrainer.validate_one_epoch)!r   r   r    r!   classmethodargparse	Namespacer   r(   ArgumentParserr.   re   r   Moduler   r   r   strr   rs   r   rP   	Optimizerr   r   r   r   r   r   r"   r   no_gradr   r   r   r   r   r$   4   sT    	

	
 hr$   )T)3r!   r   dataclassesr   rh   
contextlibr   typingr   r   r   r   r   r   re   packaging.versionr	   V	typeguardr
    espnet2.schedulers.abs_schedulerr   r    espnet2.torch_utils.device_funcsr    espnet2.torch_utils.recursive_opr   espnet2.train.distributed_utilsr   espnet2.train.reporterr   espnet2.train.trainerr   r   espnet2.utils.build_dataclassr   espnet2.utils.typesr   r_   is_availabletorch.distributedr   __version__torch.cuda.ampr   r   	fairscaleImportError	dataclassr   r$   r   r   r   r   <module>   sB    
