o iW@sdZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZdddZd d Zd d Zd dZedkrPeejdddSdS)z3Automatic speech recognition model training script.N) __version__) strtobool)BATCH_COUNT_CHOICESTcCsB|durtjdtjtjd}|jdddd|jddd d|jd dd d|jd dtd d|jddddd|jddgddd|jddtddgdd|jdt|dd |jd!d"td#d|jd$|d%d&|jd'd"td(d|jd)td*d+|jd,d-d.d/d0d1|jd2d3td4d5d6|jd7d8d9td:d|jd;dtd/dd?td@d|jdAd9tdBd|jdCtddDd6|jdEtddFd6|jdGtddHd6|jdId"tdJd|jdKdLtgdMdNd|jdOdPtdQd|jdRdStdTd|jdUdddVd|jdWdddXd|jdYtd"dZd6|jd[td\d]d6|jd^dStd_d|jd`dStdad|jdbdStdcd|jdddetdfd|jdgtddhd6|jditddjd6|jdkdltdmd|jdndotdpd|jdqdrtdsd|jdtd9td/dud=|jdvdwt dxd|jdydzd{d9td|d|jd}d9td~d|jdd9tdd|jdd9tdd|jdd9tdd|jdddtddd|jdddtddd|jdd9tdd|jdtdd/dd|jddtgddd|jdd"tdd|jddtdd|jddtdd|jddStdd|jddtgddd|jddtdd|jdddtdd|jddtd/dd=|jddtd/dd=|jddtdd|jddtdd|jddtdd|jdt ddd6|jdd"td"dgdd|jddt d/dd=|jddtdd|jddddƄdd|jddtdd|jddddƄdd|jddddƄdd|jdt ddd6|jdt ddd6|jddtgdעdd|jdtdd.d6|jdtdd.d6|jdtdd.d6|jdtdSd.d6|jdtdd.d6|jdtdd.d6|jdt ddd6|jdt dd.d6|jddtgdעdd|jdtdd.d6|jdtdd.d6|jdtdd.d6|jdtdd.d6|jdtddd6|jdtddd6|jdtdSd.d6|jdtddd6|jdt ddd6|jdt dd.d6|jdt dd.d6|jdtddd6|jdtddd6|jdtdSd.d6|jdtdd.d6|S)zGet default arguments.NzRTrain an automatic speech recognition (ASR) model on one CPU, one or multiple GPUs) descriptionconfig_file_parser_classformatter_classz--configTzconfig file path)is_config_filehelpz --config2zCsecond config file path that overwrites the settings in `--config`.z --config3zRthird config file path that overwrites the settings in `--config` and `--config2`.z--ngpuz5Number of GPUs. If not given, use all visible devices)defaulttyper z --use-ddpF store_truezEnable process-based data parallel. --ngpu's GPUs will be used. If --ngpu is not given, this tries to identify how many GPUs can be used. But, if it fails, the application will abort. And, currently, single node multi GPUs job is only supported.)r actionr z --train-dtypefloat32)float16rfloat64O0O1O2O3zData type for training (only pytorch backend). O0,O1,.. flags require apex. See https://nvidia.github.io/apex/amp.html#opt-levels)r choicesr z --backendchainerpytorchzBackend library)r r rr z--outdirzOutput directory)r requiredr z --debugmode Debugmodez--dict Dictionary)rr z--seedz Random seedz --debugdirzOutput directory for debugging)r r z--resumez-r?z!Resume the training from snapshot)r nargsr z --minibatchesz-Nz-1z&Process only N minibatches (for debug))r r r z --verbosez-VrzVerbose optionz--tensorboard-dirzTensorboard log dir path)r r rr z--report-interval-itersdzReport interval iterationsz--save-interval-itersz!Save snapshot interval iterationsz --train-jsonz#Filename of train label data (json)z --valid-jsonz(Filename of validation label data (json)z--model-modulezCmodel defined module (default: espnet.nets.xxx_backend.e2e_asr:E2E)z --num-encsz Number of encoders in the model.z --ctc_typebuiltin)r gtnctccudnnctcz-Type of CTC implementation to calculate loss.z --mtlalphag?zKMultitask learning coefficient, alpha: alpha*ctc_loss + (1-alpha)*att_loss z --lsm-weightgzLabel smoothing weightz --report-cerzCompute CER on development setz --report-werzCompute WER on development setz--nbestzOutput N-best hypothesesz --beam-sizez Beam sizez --penaltyzIncertion penaltyz --maxlenratiozInput length ratio to obtain max output length. If maxlenratio=0.0 (default), it uses a end-detect function to automatically find maximum hypothesis lengthsz --minlenratioz.Input length ratio to obtain min output lengthz --ctc-weightg333333?zCTC weight in joint decodingz--rnnlmzRNNLM model file to readz --rnnlm-confzRNNLM model config file to readz --lm-weightg?z RNNLM weight.z --sym-spacezz Space symbolz --sym-blankz Blank symbolz --sortagradzFHow many epochs to use sortagrad for. 0 = deactivated, -1 = all epochsz --batch-countautozKHow to count batch_size. The default (auto) will find how to count by args.z --batch-sizez --batch-seqsz-bz*Maximum seqs in a minibatch (0 to disable)z --batch-binsz*Maximum bins in a minibatch (0 to disable)z--batch-frames-inz2Maximum input frames in a minibatch (0 to disable)z--batch-frames-outz3Maximum output frames in a minibatch (0 to disable)z--batch-frames-inoutz9Maximum input+output frames in a minibatch (0 to disable)z --maxlen-inz--batch-seq-maxlen-ini MLzPWhen --batch-count=seq, batch size is reduced if the input sequence length > ML.)r r metavarr z --maxlen-outz--batch-seq-maxlen-outzPWhen --batch-count=seq, batch size is reduced if the output sequence length > MLz--n-iter-processeszNumber of processes of iteratorz--preprocess-confz-The configuration file for the pre-processing)r r rr z--optadadelta)r)adamnoam Optimizerz --accum-gradzNumber of gradient accumurationz--epsg:0yE>zEpsilon constant for optimizerz --eps-decayg{Gz?zDecaying ratio of epsilonz--weight-decayzWeight decay ratioz --criterionacc)lossloss_eps_decay_onlyr-z"Criterion to perform epsilon decayz --thresholdg-C6?zThreshold to stop iterationz--epochsz-ezMaximum number of epochsz--early-stop-criterionzvalidation/main/accz=Value to monitor to trigger an early stopping of the trainingz --patiencezINumber of epochs to wait without improvement before stopping the trainingz --grad-clipzGradient norm threshold to clipz--num-save-attentionz*Number of samples of attention to be savedz--num-save-ctcz0Number of samples of CTC probability to be savedz --grad-noisezFThe flag to switch to use noise injection to gradients during trainingz --num-spkrsz!Number of speakers in the speech.z--context-residualzHThe flag to switch to use context vector residual in the decoder networkz --enc-initz,Pre-trained ASR model to initialize encoder.z--enc-init-modszenc.enc.cfdddDS)Ncg|] }dkrt|qSrstr.0modsH/home/ubuntu/.local/lib/python3.10/site-packages/espnet/bin/asr_train.py y0get_parser....,splitr<r>r<r?yzget_parser..zr?r@rArBrCrDr<r>r<r?rFrGzr?r@rArBrCrDr<r>r<r?rFrGz0List of modules to freeze, separated by a comma.z--use-frontendz*The flag to switch to use frontend system.z --use-wpezApply Weighted Prediction Errorz--wtypeblstmp)lstmblstmlstmprHvgglstmp vggblstmpvgglstmvggblstmgrubgrugrupbgrupvgggrupvggbgrupvgggruvggbgruzDType of encoder network architecture of the mask estimator for WPE. z --wlayersz--wunitsi,z--wprojsz--wdropout-ratez --wpe-tapsz --wpe-delayz--use-dnn-mask-for-wpezGUse DNN to estimate the power spectrogram. This option is experimental.z--use-beamformerz--btypezJType of encoder network architecture of the mask estimator for Beamformer.z --blayersz--bunitsz--bprojsz--badimi@z--bnmaskz>Number of beamforming masks, default is 2 for [speech, noise].z --ref-channelzWThe reference channel used for beamformer. By default, the channel is estimated by DNN.z--bdropout-ratez --stats-filez,The stats file for the feature normalizationz--apply-uttmvnz2Apply utterance level mean variance normalization.z--uttmvn-norm-meansz--uttmvn-norm-varsz --fbank-fsi>z5The sample frequency used for the mel-fbank creation.z--n-melsPz!The number of mel-frequency bins.z --fbank-fminz --fbank-fmax) configargparseArgumentParserYAMLConfigFileParserArgumentDefaultsHelpFormatteradd add_argumentintr8floatrr)parserrr>r>r? get_parsersT    rccCs:|dkrtjtjdddStjtjddtddS)z*Make logging setup with a given log level.rz>%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s)levelformatzSkip DEBUG/INFO messagesN)logging basicConfigINFOWARNwarning)verboser>r>r? setup_loggings rlc Cst}||\}}|jdkr|jdkrtd|jd|jdkr0|jdvr0td|jddd lm}|j d urP|j d krHd |jd }n d |jd}n|j }||}| || |}||_ d|j vrld|_d|j vrtd|_t |_t|j|jd urtjd}|d urt|d}nr>r?r@ns zmain..r$zmaskctczz backend = z"Chainer with DDP is not supported.)launchset_start_methodspawnz0Single speaker is only supported when using DDP.trainz'Only chainer and pytorch are supported.zOnly pytorch is supported.)3rcparse_known_argsbackend train_dtypeNotImplementedErrorngpu ValueErrorespnet.utils.dynamic_importrm model_module num_spkrs add_arguments parse_argsrversionrlrkosenvirongetlenrErfrj subprocessrunPIPECalledProcessErrorFileNotFoundErrorrrrwdebuguse_ddpinfoseedrandomnpdictopen readlinesinsertappend char_list)espnet.distributed.pytorch_backend.launchrzr{$_reinitialize_logging_and_call_trainespnet.asr.chainer_backend.asrr~espnet.asr.pytorch_backend.asr"espnet.asr.pytorch_backend.asr_mix)cmd_argsrbargs_rmr model_classcvdrpf dictionaryrrzr{r~r>r>r?mains                                   rcCs"ddlm}t|j||dS)Nrr})rr~rlrk)rr~r>r>r?rs   r__main__r)NT)__doc__rfrrrsysrZnumpyrespnetrespnet.utils.cli_utilsrespnet.utils.training.batchfyrrcrlrr__name__argvr>r>r>r?s,    z