o
    i	                    @   s  d Z ddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
 ddlmZ ddlmZmZmZmZmZmZmZmZ ddlZddlZddlZddlZddlZddlZddlZddlmZ  ddl!m"Z" dd	l#m$Z$m%Z% dd
l&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z: ddl;m<Z< ddl=m>Z> ddl?m@Z@ ddlAmBZB ddlCmDZD ddlEmFZF ddlGmHZH ddlImJZJ ddlKmLZLmMZMmNZN ddlOmPZPmQZQmRZRmSZSmTZTmUZU ddlVmWZW ddlXmYZY dd lZm[Z[ dd!l\m]Z] dd"l^m_Z_ dd#l`maZa dd$lbmcZcmdZdmeZemfZfmgZgmhZh dd%limjZj dd&lkmlZl zddlmZmW n enyH   dZmY nw e ej'e d'krZdd(lompZp ndd)lomqZp erejsjtejsjue3ejsjvejsjwejsjxejsjyejsjzejsj{ejsj|d*
Z}e ej'e d+kre}j~ejsjd, z1ddlZe}j~ejejejejejejejejejejd-
 e ej'e d.k re}j~ejd, [W n
 ey   Y nw zddlZe}j~ejjejjejjejjd/ [W n
 ey   Y nw zddlZW n ey   dZY nw erejsjjejsjjejsjjejsjjejsjjejsjje:e>e<ejsjjejsjjejsjjd0Zd1d2 e} D Z}d3d2 e D Ze
G d4d5 d5ZG d6d7 d7eZdS )8zAbstract task module.    N)ABCabstractmethod)	dataclass)Path)AnyCallableDictListOptionalSequenceTupleUnion)parse)
DataLoader)check_argument_typescheck_return_type)__version__)AbsIterFactory)ChunkIterFactory)MultipleIterFactory)SequenceIterFactory)collect_stats)SGD)BATCH_TYPESbuild_batch_sampler)UnsortedBatchSampler)NoamLR)WarmupLR)WarmupStepLR)load_pretrained_model)model_summary)pytorch_cudnn_version)set_all_random_seed)AbsESPnetModel)ClassChoices)
DATA_TYPES
AbsDatasetESPnetDataset)DistributedOption	free_portget_master_portget_node_rankget_num_nodesresolve_distributed_mode)IterableESPnetDataset)Trainer)config_argparse)build_dataclass)get_default_kwargs)NestedDictAction) humanfriendly_parse_size_or_noneint_or_nonestr2boolstr2triple_str
str_or_intstr_or_none)yaml_no_alias_safe_dump)get_commandline_argsz1.5.0)ProcessContext)SpawnContext)
adamadamwsgdadadeltaadagradadamaxasgdlbfgsrmsproprpropz1.10.0)radam)
accagdadaboundadamoddiffgradlambnovogradpidqhmsgdwyogiz0.2.0)	fusedadam	fusedlambfusednovogradfusedsgd)ReduceLROnPlateaulambdalrsteplrmultisteplrexponentiallrCosineAnnealingLRnoamlrwarmupsteplrwarmuplrcycliclr
onecyclelrCosineAnnealingWarmRestartsc                 C      i | ]	\}}|  |qS  lower.0kvrd   rd   J/home/ubuntu/.local/lib/python3.10/site-packages/espnet2/tasks/abs_task.py
<dictcomp>       rl   c                 C   rc   rd   re   rg   rd   rd   rk   rl      rm   c                   @   s~   e Zd ZU eed< eed< eed< eed< eed< eed< eed< eed< eed	< e	ed
< e
e ed< e
e ed< e	ed< dS )IteratorOptionspreprocess_fn
collate_fndata_path_and_name_and_typeshape_files
batch_size
batch_bins
batch_typemax_cache_sizemax_cache_fddistributednum_batchesnum_iters_per_epochtrainN)__name__
__module____qualname__callable__annotations__listintstrfloatboolr
   rd   rd   rd   rk   rn      s   
 rn   c                   @   s|  e Zd ZU dZeed< eZg Ze	e
 ed< dd ZeedejfddZeed	ejd
edeeeeejf  geeejf f fddZeed	ejd
edeeeeeejf geeejf f  fddZee	dOd
ededeedf fddZ ee	dOd
ededeedf fddZ!eed	ejde"fddZ#ede$jfddZ%ed	ejdej&j'de	ej(j) fddZ*edeedf fdd Z+edeee,f fd!d"Z-ed	ejfd#d$Z.e	dPd%e/e0e1f d&ed
ededd'f
d(d)Z2ee3j4fdQd*d+Z5edRd	ejd,ee fd-d.Z6ed	ejfd/d0Z7ed	ejd1e8d2efd3d4Z9e	'dSd	ejd1e8d2ed5e:de;f
d6d7Z<ed	ejd8e=d2ede;fd9d:Z>ed	ejd8e=d2ede;fd;d<Z?ed	ejd8e=d2ede;fd=d>Z@ed	ejd1e8d2efd?d@ZAed'dejBdddAdfdBedCedDedEed&edFededeCfdGdHZDe	'	'	IdTdJe/eEef dKe/eEef dLedee"ejf fdMdNZFd'S )UAbsTask   num_optimizersclass_choices_listc                 C   s   t d)Nz!This class can't be instantiated.)RuntimeError)selfrd   rd   rk   __init__   s   zAbsTask.__init__parserc                 C   s   d S Nrd   )clsr   rd   rd   rk   add_task_arguments      zAbsTask.add_task_argumentsargsr{   returnc                 C      t )a  Return "collate_fn", which is a callable object and given to DataLoader.

        >>> from torch.utils.data import DataLoader
        >>> loader = DataLoader(collate_fn=cls.build_collate_fn(args, train=True), ...)

        In many cases, you can use our common collate_fn.
        NotImplementedErrorr   r   r{   rd   rd   rk   build_collate_fn   s   zAbsTask.build_collate_fnc                 C   r   r   r   r   rd   rd   rk   build_preprocess_fn   s   zAbsTask.build_preprocess_fnTF	inference.c                 C   r   )a  Define the required names by Task

        This function is used by
        >>> cls.check_task_requirements()
        If your model is defined as following,

        >>> from espnet2.train.abs_espnet_model import AbsESPnetModel
        >>> class Model(AbsESPnetModel):
        ...     def forward(self, input, output, opt=None):  pass

        then "required_data_names" should be as

        >>> required_data_names = ('input', 'output')
        r   r   r{   r   rd   rd   rk   required_data_names      zAbsTask.required_data_namesc                 C   r   )a  Define the optional names by Task

        This function is used by
        >>> cls.check_task_requirements()
        If your model is defined as follows,

        >>> from espnet2.train.abs_espnet_model import AbsESPnetModel
        >>> class Model(AbsESPnetModel):
        ...     def forward(self, input, output, opt=None):  pass

        then "optional_data_names" should be as

        >>> optional_data_names = ('opt',)
        r   r   rd   rd   rk   optional_data_names   r   zAbsTask.optional_data_namesc                 C   r   r   r   )r   r   rd   rd   rk   build_model   r   zAbsTask.build_modelc                 C   s  t  sJ G dd dtjtj}tjd|d}|jdgd |d}|jdd	d
d |jddd dddd |jdt	ddd |jdt
g dddd |jdtd d |jdtdd d |jd!tdd"d |jd#td$d%d |jd&td'd(d |d)}|jd*d+t
d,d- |jd.t
d/d0d |jd1d td2d- |jd3td d4d |jd5td d6d |jd7d td8d- |jd9d td:d- |jd;d tg d<d=d> |jd?dt	d@d- |jdAt	ddBd |jdCdt	dDd- |dE}|jdFt	tjjjdGd |jdHt	tjjjdId |jdJt	dKdLd |dM}|jdNt	ddOd |jdPt	ddQd |dR}|jdStdTdUd |jdVtd dWd |jdXt
dYdZd[d\ |jd]t
d'd^d_d\ |jd`tdag dbdcd\ |jddtdadegdfd\ |jdgtddhd |jditdjdkd |jdltdmdnd |jdot	ddpd |jdqtd$drd |jdst	ddtd |jdut	ddvd |jdwdxg dydzd{ |jd|t	dd}d |jd~td dd |jdt	dKdd |jdt	dKdd |jdt	ddd |jdt	ddd |jdt
d dd |jdt
d dd |jdt
d dd |jdt
d dd |jdtddd |jdt	ddd |d}|jddd |jdt
g ddd |jdt	ddd |jdt
g ddd |d}|jdtd dd |jdtddd |jdtd dd |jdtddd |jdtd dd |jdt
dg d |jdt
dg d |d}d}t D ]\}}|d| d| d7 }q|jdt
dtt|d |jdtd ttd g dd |jdtdg d |jdt
dddgdd |jdt
dddgdd |jdt	ddd |dġ}|jdtddd |jdtddd |jdtddd |dΡ}d}t D ]\}}|d| d|d  dѝ7 }q|jdtdg |dӍ |jdtdg d |jdt	ddd |jdtjddd |jdtddd |jdtd dd |dߡ}td$| jd$ D ]O}	|	d$krrdnt
|	}
|jd|
 dd dttdd |jd|
 dt t! dd |jd|
 dd d tt"d g dd |jd|
 dt t! dd qi| j#$| | %| t&|sJ |S )Nc                   @   s   e Zd ZdS )z@AbsTask.get_parser.<locals>.ArgumentDefaultsRawTextHelpFormatterN)r|   r}   r~   rd   rd   rd   rk   $ArgumentDefaultsRawTextHelpFormatter  s    r   zbase parser)descriptionformatter_class
output_dir)requiredzCommon configurationz--print_config
store_truezPrint the config file and exit)actionhelpz--log_levelc                 S      |   S r   )upperxrd   rd   rk   <lambda>"      z$AbsTask.get_parser.<locals>.<lambda>INFO)ERRORWARNINGr   DEBUGNOTSETzThe verbose level of logging)typedefaultchoicesr   z	--dry_runFz Perform process without training)r   r   r   z--iterator_type)sequencechunktasknoner   zSpecify iterator type)r   r   r   r   z--output_dir)r   r   z--ngpur   z(The number of gpus. 0 indicates CPU modez--seedzRandom seedz--num_workersr   z)The number of workers used for DataLoaderz--num_att_plot   zThe number images to plot the outputs from attention. This option makes sense only when attention-based model. We can also disable the attention plot by setting it 0zdistributed training relatedz--dist_backendncclzdistributed backend)r   r   r   z--dist_init_methodenv://zkif init_method="env://", env values of "MASTER_PORT", "MASTER_ADDR", "WORLD_SIZE", and "RANK" are referred.z--dist_world_sizez(number of nodes for distributed trainingz--dist_rankz"node rank for distributed trainingz--local_rankz_local rank for distributed training. This option is used if --multiprocessing_distributed=falsez--dist_master_addrzaThe master address for distributed training. This value is used when dist_init_method == 'env://'z--dist_master_portz\The master port for distributed trainingThis value is used when dist_init_method == 'env://'z--dist_launcher)slurmmpiNz*The launcher type for distributed training)r   r   r   r   z--multiprocessing_distributedzUse multi-processing distributed training to launch N processes per node, which has N GPUs. This is the fastest way to use PyTorch for either single node or multi node data parallel trainingz--unused_parameterszWWhether to use the find_unused_parameters in torch.nn.parallel.DistributedDataParallel z--sharded_ddpz-Enable sharded training provided by fairscalezcudnn mode relatedz--cudnn_enabledzEnable CUDNNz--cudnn_benchmarkzEnable cudnn-benchmark modez--cudnn_deterministicTzEnable cudnn-deterministic modezcollect stats mode relatedz--collect_statszPerform on "collect stats" modez--write_collected_featszBWrite the output features from the model when "collect stats" modezTrainer relatedz--max_epoch(   z!The maximum number epoch to trainz
--patiencezINumber of epochs to wait without improvement before stopping the trainingz--val_scheduler_criterion   )validlosszThe criterion used for the value given to the lr scheduler. Give a pair referring the phase, "train" or "valid",and the criterion name. The mode specifying "min" or "max" can be changed by --scheduler_conf)r   nargsr   r   z--early_stopping_criterionr   r   minzThe criterion used for judging of early stopping. Give a pair referring the phase, "train" or "valid",the criterion name and the mode, "min" or "max", e.g. "acc,max".z--best_model_criterion+))r{   r   r   r   )r{   accmax)r   r   r   zThe criterion used for judging of the best model. Give a pair referring the phase, "train" or "valid",the criterion name, and the mode, "min" or "max", e.g. "acc,max".z--keep_nbest_models
   z<Remove previous snapshots excluding the n-best scored epochsz--nbest_averaging_intervalzAThe epoch interval to apply model averaging and save nbest modelsz--grad_clipg      @zGradient norm threshold to clipz--grad_clip_typeg       @z9The type of the used p-norm for gradient clip. Can be infz--grad_noisezFThe flag to switch to use noise injection to gradients during trainingz--accum_gradz#The number of gradient accumulationz--no_forward_runzFJust only iterating data loading without model forwarding and trainingz--resumez)Enable resuming if checkpoint is existingz--train_dtypefloat32)float16r   float64zData type for training.)r   r   r   z	--use_ampzDEnable Automatic Mixed Precision. This feature requires pytorch>=1.6z--log_intervalzShow the logs every the number iterations in each epochs at the training phase. If None is given, it is decided according the number of training samples automatically .z--use_matplotlibzEnable matplotlib loggingz--use_tensorboardzEnable tensorboard loggingz--create_graph_in_tensorboardz&Whether to create graph in tensorboardz--use_wandbzEnable wandb loggingz--wandb_projectzSpecify wandb projectz
--wandb_idzSpecify wandb idz--wandb_entityzSpecify wandb entityz--wandb_namezSpecify wandb run namez--wandb_model_log_intervalzSet the model log periodz--detect_anomalyz%Set torch.autograd.set_detect_anomalyzPretraining model relatedz--pretrain_pathzThis option is obsoleted)r   z--init_param*a  Specify the file path used for initialization of parameters. The format is '<file_path>:<src_key>:<dst_key>:<exclude_keys>', where file_path is the model file path, src_key specifies the key of model states to be used in the model file, dst_key specifies the attribute of the model to be initialized, and exclude_keys excludes keys of model states for the initialization.e.g.
  # Load all parameters  --init_param some/where/model.pth
  # Load only decoder parameters  --init_param some/where/model.pth:decoder:decoder
  # Load only decoder parameters excluding decoder.embed  --init_param some/where/model.pth:decoder:decoder:decoder.embed
  --init_param some/where/model.pth:decoder:decoder:decoder.embed
)r   r   r   r   z--ignore_init_mismatchz3Ignore size mismatch when loading pre-trained modelz--freeze_paramzFreeze parameterszBatchSampler relatedz--num_iters_per_epochz8Restrict the number of iterations for training per epochz--batch_size   z\The mini-batch size used for training. Used if batch_type='unsorted', 'sorted', or 'folded'.z--valid_batch_sizez/If not given, the value of --batch_size is usedz--batch_binsi@B z@The number of batch bins. Used if batch_type='length' or 'numel'z--valid_batch_binsz/If not given, the value of --batch_bins is usedz--train_shape_fileappend)r   r   r   z--valid_shape_filezSequence iterator related "z":

z--batch_typefoldedz--valid_batch_typez/If not given, the value of --batch_type is usedz--fold_lengthz--sort_in_batch
descending	ascendingz{Sort the samples in each mini-batches by the sample lengths. To enable this, "shape_file" must have the length information.z--sort_batchz'Sort mini-batches by the sample lengthsz--multiple_iteratorzUse multiple iterator modezChunk iterator relatedz--chunk_lengthi  aP  Specify chunk length. e.g. '300', '300,400,500', or '300-400'.If multiple numbers separated by command are given, one of them is selected randomly for each samples. If two numbers are given with '-', it indicates the range of the choices. Note that if the sequence length is shorter than the all chunk_lengths, the sample is discarded. z--chunk_shift_ratiog      ?zSpecify the shift width of chunks. If it's less than 1, allows the overlapping and if bigger than 1, there are some gaps between each chunk.z--num_cache_chunksi   z|Shuffle in the specified number of chunks and generate mini-batches More larger this value, more randomness can be obtained.zDataset relateda  Give three words splitted by comma. It's used for the training data. e.g. '--train_data_path_and_name_and_type some/path/a.scp,foo,sound'. The first value, some/path/a.scp, indicates the file path, and the second, foo, is the key name used for the mini-batch data, and the last, sound, decides the file type. This option is repeatable, so you can input any number of features for your task. Supported file types are as follows:

r   z

z#--train_data_path_and_name_and_type)r   r   r   r   z#--valid_data_path_and_name_and_typez--allow_variable_data_keyszKAllow the arbitrary keys for mini-batch with ignoring the task requirementsz--max_cache_size        z8The maximum cache size for data loader. e.g. 10MB, 20GB.z--max_cache_fd    zThe maximum number of file descriptors to be kept as opened for ark files. This feature is only valid when data type is 'kaldi_ark'.z--valid_max_cache_sizezsThe maximum cache size for validation data loader. e.g. 10MB, 20GB. If None, the 5 percent size of --max_cache_sizezOptimizer relatedz--optimc                 S   r   r   re   r   rd   rd   rk   r   8  r   rA   zThe optimizer type_confz#The keyword arguments for optimizer)r   r   r   z--schedulerc                 S   s   t |  S r   )r9   rf   r   rd   rd   rk   r   E  s    zThe lr scheduler typez&The keyword arguments for lr scheduler)'r   argparseRawTextHelpFormatterArgumentDefaultsHelpFormatterr0   ArgumentParserset_defaultsadd_argument_groupadd_argumentr6   r   r9   r   r5   torchbackendscudnnenabled	benchmarkr7   r   r   itemsr   r8   r%   humanfriendly
parse_sizer4   ranger   optim_classesr3   dictscheduler_classestraineradd_argumentsr   r   )r   r   r   group_batch_type_helpkeyvalue!_data_path_and_name_and_type_helpdicisufrd   rd   rk   
get_parser  s  




		



	




	




zAbsTask.get_parsermodelc                 C   s   | j dkr	tdt|j}|d u r tdtt d|j |jr<td u r+tdtjj	j
d| |d|j}n|| fi |j}|g}|S )Nr   z<build_optimizers() must be overridden if num_optimizers != 1must be one of : z/Requiring fairscale. Do 'pip install fairscale')paramsoptimrd   )r   r   r   getr   
ValueErrorr   sharded_ddp	fairscaleossOSS
parameters
optim_conf)r   r   r   optim_classr   
optimizersrd   rd   rk   build_optimizersW  s$   

zAbsTask.build_optimizersc                 C   s   dS )z-The options not to be shown by --print_config)r   print_configconfigngpurd   )r   rd   rd   rk   exclude_optsq  r   zAbsTask.exclude_optsc                 C   sp  dt dtfdd}t sJ |  }| \}}t|}t D ]}|| q t	d| j
d D ]V}|dkr8dnt |}|d|  }	||	t}
t|
}||d| d  ||d| d< |d	|  }	|	d
ur||	t}t|}||d	| d  ||d	| d< q0| jD ]+}t||jd
ur|t||j}t|}|j}	|||	 d  |||	 d< q|S )zYReturn the configuration as dict.

        This method is used by print_config()
        nameclassesc                 S   s.   | | }|d u rtdt| d|  |S )Nr   r   )r   r   r   )r  r  _clsrd   rd   rk   get_class_type}  s   
z2AbsTask.get_default_config.<locals>.get_class_typer   r   r   r   	schedulerN)r   r   r   r   parse_known_argsvarsr   r  popr   r   r   r2   updater   r   getattrr  	get_class)r   r  r   r   _r	  ri   r   r   r  r  confscheduler_classclass_choices	class_objrd   rd   rk   get_default_configv  s<   



zAbsTask.get_default_configc                    s   t  sJ t D ]}d|v rtd| dq	d fdd jD }t|dkrW|  }|jtj	d t
tjd j}ttj	d t| d	| tj	d td
 d S d S )N-z+Use "_" instead of "-": parser.get_parser("z")z, c                 3   s(    | ]}t  |d u rd| V  qd S )Nz--)r  )rh   ar   rd   rk   	<genexpr>  s    z6AbsTask.check_required_command_args.<locals>.<genexpr>r   filez/: error: the following arguments are required: r   )r   r  r   joinr   lenr   
print_helpsysstderrr   argvr  printexit)r   r   ri   r   r   prd   r  rk   check_required_command_args  s&   
z#AbsTask.check_required_command_argsdatasetallow_variable_data_keysNc              
   C   s   t  sJ d| j d| j d}| ||D ]}||s4td| || d| j d|  d| q|s_| ||| || }| D ]}||vr^td| d	| j d
| d| qGdS dS )z<Check if the dataset satisfy the requirement of current Taskz2If you intend to use an additional input, modify "z.required_data_names()" or "zS.optional_data_names()". Otherwise you need to set --allow_variable_data_keys true r   z" are required for z. but "z" are input.
zThe data-name must be one of z for z: "z" is not allowed.
N)r   r|   r   has_namer   namesr   )r   r-  r.  r{   r   mesri   	task_keysrd   rd   rk   check_task_requirements  sN   
	

zAbsTask.check_task_requirementsc                 C   s*   t  sJ |  }|t|ddd d S )N   Findent	sort_keys)r   r  writer:   )r   r"  r	  rd   rd   rk   r    s   
zAbsTask.print_configcmdc                 C   s  t  sJ tt tjd |d u r|  }||}t|_|j	d ur&t
d|jr2|   td | | t| |jrA|jsH| | d S |jdksRJ |jt|j|j}|dkrsd|_d|_|jdkrst|jd u rst |_|j| |_t|j|j}g }g }tjd}t |jD ]4}	t!j"d
i t#|}
|	|
_$|j| |	 |
_d|
_|j%| j|
fdd	}|&  |'| |'|(  qt)||* s	 t)||* rd S d S )Nr!  z/--pretrain_path is deprecated. Use --init_paramr   r   	localhostr   spawnF)targetr   daemonrd   )+r   r)  r;   r&  r'  r   
parse_argsr   versionpretrain_pathr   r  r*  r,  r-   rx   multiprocessing_distributedmain_workerr
  r,   dist_world_sizedist_launcherdist_master_addr	dist_rankdist_init_methodr*   dist_master_portr)   r+   r   multiprocessingget_contextr   r   	Namespacer  
local_rankProcessstartr   SimpleQueuer<   r#  )r   r   r9  r   	num_nodes	node_rankerror_queues	processesmpr   
local_argsprocessrd   rd   rk   main  sX   




	

zAbsTask.mainc                 C   sb  t  sJ tt|}|  |jr|jdkr>|jsd}n
d|j d|j }tj|j	dt
 d dd  | dd	 ntjd
dt
 d dd  d|j d|j dd	 |  t|j |jtjj_|jtjj_|jtjj_|jrtd tj|j | j|d}t|tst dtj! dt"| |j#t$t|j%|j&dkrdndd}|j'D ]"}|( D ]\}}|)|d s||krtd| d d|_*qq| j+||d}g }	t,|dD ]J\}
}|
dkrdnt-|
}t$|d| }t$|d| d}|d ur%t./|}|d u rt0dt1t. d| ||fi |}nd }|	2| qtt3  tt4| t,t5||	dD ](\}
\}}|
dkrOdnt-|
}td| d|  td| d|  qBt6|j7}|jrz|jdkr|j8ddd |d j9d d!d"}td#|d   t:t;||d$dd% W d    n	1 sw   Y  |j<rd S |j=r0t| |j>d u r|j?|_>t@|jAdkr|jAd }nd }t@|jBdkr|jBd }nd }t=|| jC|jD||j?|j%|jE|jF|j&| jG|dd&| jH|dd&d'	| jC|jI||j>|j%|jE|jF|j&| jG|dd&| jH|dd&d'	||j&|jJ|jKd( d S |jLD ]"}td)|  tM|||jN|j&dkrPd*tjOP  ndd+ q3|jQrc| jR||d,d-}n| jS||d,d-}| jS||d.d-}|jTs|jUdkrd|_Utd/ |jUdkr| jS||d0d-}nd }|jVrtWd u rt d1ztWX  W n tWjYjZy   td2 d|_VY nw |jVr|jr|jdkr|j[d u rd3| j! }n|j[}|j\d u rt-t6d] ^dd4}n|j\}tWj_|j`||t-||ja|jbd5 tWjcd| nd|_V| jef|}| jejg|||	|||||d6 |jVr-tWjgr/tWh  d S d S d S )7Nr   r   :/[r   .z@] %(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s)levelformatr   z0Invoking torch.autograd.set_detect_anomaly(True)r  model must inherit 
, but got cudacpu)dtypedevicezSetting z.requires_grad = FalseF)r   r  r   r   r   	Optimizerz:
	SchedulerT)parentsexist_okconfig.yamlwutf-8encodingzSaving the configuration in r4  r5  r{   )	rq   key_filers   rb  num_workersr.  r
  ro   rp   )r   
train_iter
valid_iterr   r
  log_intervalwrite_collected_featszLoading pretrained params from cuda:)r   
init_paramignore_init_mismatchmap_locationr{   )r   distributed_optionmoder   z6--use_matplotlib false => Changing --num_att_plot to 0plot_attzPlease install wandbz1wandb not configured! run `wandb login` to enableESPnet_r  )entityprojectr  diridresume)r   r  
schedulerstrain_iter_factoryvalid_iter_factoryplot_attention_iter_factorytrainer_optionsrx  )ir   r1   r(   init_optionsrx   rF  rC  loggingbasicConfig	log_levelosunamesplitinit_torch_distributedr"   seedcudnn_enabledr   r   r   r   cudnn_benchmarkr   cudnn_deterministicdeterministicdetect_anomalyinfoautogradset_detect_anomalyr   
isinstancer#   r   r|   r   tor  train_dtyper
  freeze_paramnamed_parameters
startswithrequires_gradr  	enumerater   r   r   r   r   r   r!   r    zipr   r   mkdiropenr:   r  dry_runr   valid_batch_sizers   r$  train_shape_filevalid_shape_filebuild_streaming_iterator!train_data_path_and_name_and_typero  r.  r   r   !valid_data_path_and_name_and_typerr  rs  ru  r   rv  r`  current_devicemultiple_iteratorbuild_multiple_iter_factorybuild_iter_factoryuse_matplotlibnum_att_plot	use_wandbwandbloginerrors
UsageErrorwandb_project
wandb_nameresolvereplaceinitwandb_entitywandb_idr  r	  r  r   build_optionsrunfinish)r   r   rx  _rankr   tri   r+  r  r  r   r   r   r  r  cls_r  osr   ftrain_key_filevalid_key_filer  r  r  r}  r  rd   rd   rk   rB  0  s  


	 
















zAbsTask.main_workerrx  ry  c                 C   s  |dkr2| j |dd}| j|dd}|j}|j}|j}|j}	|j}
|j}|j}|j	}d }|j
}d}n|dkr| j |dd}| j|dd}|j}|j}|jd u rS|j}
n|j}
|jd u r_|j}n|j}|jd u rk|j}	n|j}	|jd u ryd|j }n|j}|j}|j	}d }d }d}n<|dkr| j |dd}| j|dd}|j}|j}d}
d	}d
}	|j}|j}|jdkrtjnd}d}d }d}ntd| t|||||
||	||||||dS )Nr{   Trm  r   Fg?rz  unsortedr   r   r   zmode=)ro   rp   rq   rr   ru   rs   rt   ry   rv   rw   rx   rz   r{   )r   r   r  r  rs   rt   ru   rv   rw   rx   rz   r  r  valid_batch_typer  valid_batch_binsvalid_max_cache_sizer  npinfr   rn   )r   r   rx  ry  ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   rd   rd   rk   build_iter_options1  s   



zAbsTask.build_iter_optionskwargsc                 C   s   t  sJ | |||}|dur| D ]
\}}t||| q|jdkr,| j|||dS |jdkr9| j|||dS |jdkrF| j|||dS td|j )a6  Build a factory object of mini-batch iterator.

        This object is invoked at every epochs to build the iterator for each epoch
        as following:

        >>> iter_factory = cls.build_iter_factory(...)
        >>> for epoch in range(1, max_epoch):
        ...     for keys, batch in iter_fatory.build_iter(epoch):
        ...         model(**batch)

        The mini-batches for each epochs are fully controlled by this class.
        Note that the random seed used for shuffling is decided as "seed + epoch" and
        the generated mini-batches can be reproduces when resuming.

        Note that the definition of "epoch" doesn't always indicate
        to run out of the whole training corpus.
        "--num_iters_per_epoch" option restricts the number of iterations for each epoch
        and the rest of samples for the originally epoch are left for the next epoch.
        e.g. If The number of mini-batches equals to 4, the following two are same:

        - 1 epoch without "--num_iters_per_epoch"
        - 4 epoch with "--num_iters_per_epoch" == 4

        Nr   )r   iter_optionsry  r   r   zNot supported: iterator_type=)	r   r  r   setattriterator_typebuild_sequence_iter_factorybuild_chunk_iter_factorybuild_task_iter_factoryr   )r   r   rx  ry  r  r  ri   rj   rd   rd   rk   r    s0   
 


zAbsTask.build_iter_factoryr  c           
         s  t  sJ t|j|j|j|j|jd}| j||j|j	d t
t
|jd d jd rCtt
t
|jd d jd}td|  nd }t|j|j|j|j|j|j|jd|jr]tj nd|d
}t|}|jd urr|d |j }d	d
 |D }td| d|  td| d|  td| dt| dt |ddt!| dt"| 
 |jrtj tj#  |D ]}	t|	k rt$dt|	 d q fdd
|D }t%|||j&|j'|j	|j(|j)|j*dkdS )Nfloat_dtype
preprocessrv   rw   rm  r   utt2categoryzReading Fr   )
r   rr   fold_lengthsrs   rt   sort_in_batch
sort_batch	drop_lastmin_batch_sizeutt2category_filec                 S   s   g | ]}t |qS rd   )r$  rh   batchrd   rd   rk   
<listcomp>  s    z7AbsTask.build_sequence_iter_factory.<locals>.<listcomp>rZ  ] dataset:
z] Batch sampler: z$] mini-batch sizes summary: N-batch=z, mean=z.1fz, min=z, max=z6The batch-size must be equal or more than world_size: z < c                    s   g | ]	}| d  qS r   rd   r  rank
world_sizerd   rk   r    rm   )r-  batchesr  rz   shufflero  rp   
pin_memory)+r   r'   rq   r  ro   rv   rw   r3  r.  r{   r   parentexistsr   r  warningr   ru   rr   fold_lengthrs   rt   r  r  rx   r   get_world_sizer   ry   r  r$  r  meanr   r   get_rankr   r   r  rz   ro  rp   r
  )
r   r   r  ry  r-  r  batch_samplerr  bs_listr  rd   r  rk   r    s   





z#AbsTask.build_sequence_iter_factoryc                 C   sf  t  sJ t|j|j|j|j|jd}| j||j|j	d t
|jdkr,|jd d }n|jd }td|d}t|}|jd urG|d |j }td| d|  |jrtj }tj }	t
||k ritd|j|k rrtd	|	|j| k r|j| d }
n|j| }
|j| }||	d | }n|j}
|j}t|||j|
|j|j	|j|j|jdk|j|j|d
S )Nr  rm  r   r   )rs   rn  rZ  r  z,Number of samples is smaller than world_sizez0batch_size must be equal or more than world_size)r-  r  r  rs   num_samples_per_epochr  ro  rp   r  chunk_lengthchunk_shift_rationum_cache_chunks) r   r'   rq   r  ro   rv   rw   r3  r.  r{   r$  rr   r   r   ry   r  r  rx   r   r  r  r   rs   r  r   r  rz   ro  rp   r
  r  r  )r   r   r  ry  r-  rn  r  r  r  r  rs   r  rd   rd   rk   r    s^   








z AbsTask.build_chunk_iter_factoryc                 C   r   )at  Build task specific iterator factory

        Example:

            >>> class YourTask(AbsTask):
            ... @classmethod
            ... def add_task_arguments(cls, parser: argparse.ArgumentParser):
            ...     parser.set_defaults(iterator_type="task")
            ...
            ... @classmethod
            ... def build_task_iter_factory(
            ...     cls,
            ...     args: argparse.Namespace,
            ...     iter_options: IteratorOptions,
            ...     mode: str,
            ... ):
            ...     return FooIterFactory(...)
            ...
            ... @classmethod
            ... def build_iter_options(
            ....    args: argparse.Namespace,
            ...     distributed_option: DistributedOption,
            ...     mode: str
            ... ):
            ...     # if you need to customize options object
        r   )r   r   r  ry  rd   rd   rk   r  \  s   !zAbsTask.build_task_iter_factoryc              	      s  t  sJ  tjdksJ tjd dd jD tj D ]n}t| s8t| dt|d }|	 sIt
| d| '}t| }d urk|krktdjd d  d| |W d    n1 sww   Y  tD ]}t|d	|  }|	 st
| dqq)fd
dtD }	fddtD }
fddtD }j  fddt|	|
|D }t|j jdS )Nr   c                 S   s   g | ]\}}}|qS rd   rd   )rh   pathr  rd   rd   rk   r    s    
z7AbsTask.build_multiple_iter_factory.<locals>.<listcomp>z is not a directory
num_splitsz is not foundz Number of splits are mismathed: z and split.c                        g | ]  fd dj D qS )c                    s.   g | ]\}}}t t|d    ||fqS r  r   r   )rh   r+  nr  r   rd   rk   r    s    BAbsTask.build_multiple_iter_factory.<locals>.<listcomp>.<listcomp>)rq   rh   r  r  rk   r    s    
c                    r  )c                    s"   g | ]}t t|d    qS r  r  )rh   r  r  rd   rk   r    s   " r  )rr   r  r  r  rk   r    s    c                    s(   g | ]} j d ur j |  nd qS r   )rz   )rh   r   )r  r  rd   rk   r    s    
c                    s4   g | ]\}}}t jj t|||d dqS ))rq   rr   rz   rv   )r  )	functoolspartialr  r   )rh   _data_path_and_name_and_type_shape_files_num_iters_per_epoch)r   r   rx  rv   ry  rd   rk   r    s$    )build_funcsr  r  )r   r  r$  rq   r   rr   r   is_dirr   r  FileNotFoundErrorr  r   readr   rv   r  r   r{   r  )r   r   rx  ry  r  r+  r  _num_splitsr    data_path_and_name_and_type_listshape_files_listnum_iters_per_epoch_listr  rd   )r   r   rx  r  rv   ry  r  rk   r    sl   

	



z#AbsTask.build_multiple_iter_factoryr   rn  rs   rb  ro  r
  c                 C   s~   t  sJ |durt|d}ni }t||||d}|jr#|jdd n|j|d | j||d|
d td
||	dk|d	|S )z'Build DataLoader using iterable datasetN)rp   )r  r  rn  r   )rs   F)r{   r   r   )r-  r  ro  rd   )r   r   r.   apply_utt2categoryr  r3  r   )r   rq   ro   rp   rn  rs   rb  ro  r.  r
  r   r  r-  rd   rd   rk   r    s.   
z AbsTask.build_streaming_iteratorra  config_file
model_filerc  c                 C   s   t  sJ |du r|dusJ dt|jd }nt|}|jddd}t|}W d   n1 s4w   Y  tjdi |}| |}t	|t
sXtdt
j dt| || |durw|d	krmd
tj  }|tj||d ||fS )a$  Build model from the files.

        This method is used for inference or fine-tuning.

        Args:
            config_file: The yaml file saved when training.
            model_file: The model file saved when training.
            device: Device type, "cpu", "cuda", or "cuda:N".

        NzZThe argument 'model_file' must be provided if the argument 'config_file' is not specified.rh  rrj  rk  r^  r_  r`  rt  )rw  rd   )r   r   r  r  yaml	safe_loadr   rK  r   r  r#   r   r|   r   r  r   r`  r  load_state_dictload)r   r  r  rc  r  r   r   rd   rd   rk   build_model_from_file   s,   




zAbsTask.build_model_from_file)TF)F)r   N)NNr   )NNra  )Gr|   r}   r~   r   r   r   r/   r   r   r	   r$   r   classmethodr   r   r   r   rK  r   r   r   r   r   r  ndarrayr   Tensorr   r
   arrayr   r   r   r   r#   r   r0   r   nnModuler   rd  r  r  r   r  r,  r   r&   r.   r3  r&  stdoutr  rW  rB  r(   r  r   r   r  rn   r  r  r  r  r   r   r  r   r  rd   rd   rd   rk   r      sv  
 $&

    W
3
!F  U<NH"S	
,

r   )__doc__r   r   r  r  r&  abcr   r   dataclassesr   pathlibr   typingr   r   r   r	   r
   r   r   r   r   numpyr  r   torch.multiprocessingtorch.nntorch.optimr  packaging.versionr   Vtorch.utils.datar   	typeguardr   r   espnetr   "espnet2.iterators.abs_iter_factoryr   $espnet2.iterators.chunk_iter_factoryr   'espnet2.iterators.multiple_iter_factoryr   'espnet2.iterators.sequence_iter_factoryr    espnet2.main_funcs.collect_statsr   espnet2.optimizers.sgdr   $espnet2.samplers.build_batch_samplerr   r   'espnet2.samplers.unsorted_batch_samplerr   espnet2.schedulers.noam_lrr   espnet2.schedulers.warmup_lrr   !espnet2.schedulers.warmup_step_lrr   )espnet2.torch_utils.load_pretrained_modelr   !espnet2.torch_utils.model_summaryr    #espnet2.torch_utils.pytorch_versionr!   'espnet2.torch_utils.set_all_random_seedr"   espnet2.train.abs_espnet_modelr#   espnet2.train.class_choicesr$   espnet2.train.datasetr%   r&   r'   espnet2.train.distributed_utilsr(   r)   r*   r+   r,   r-   espnet2.train.iterable_datasetr.   espnet2.train.trainerr/   espnet2.utilsr0   espnet2.utils.build_dataclassr1    espnet2.utils.get_default_kwargsr2    espnet2.utils.nested_dict_actionr3   espnet2.utils.typesr4   r5   r6   r7   r8   r9   %espnet2.utils.yaml_no_alias_safe_dumpr:   espnet.utils.cli_utilsr;   r  	Exceptiontorch.multiprocessing.spawnr<   r=   r   r   AdamAdamWAdadeltaAdagradAdamaxASGDLBFGSRMSpropRpropr   r  RAdamtorch_optimizerAccSGDAdaBoundAdaModDiffGradLambNovoGradPIDQHMSGDWYogiImportErrorapexr  	FusedAdam	FusedLAMBFusedNovoGradFusedSGDr   lr_schedulerrW   LambdaLRStepLRMultiStepLRExponentialLRr\   CyclicLR
OneCycleLRrb   r   r   rn   r   rd   rd   rd   rk   <module>   s   (  