o :iT@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddl mZddlmZmZmZmZmZddlZddlmZddlmZdd lmZdd lmZdd lmZe ej!d rydd l"m#Z#ndd l$m#Z#ee%efZ&Gddde'Z(Gdddej Z)e dYddZ*dYddZ+     dZddZ,ddZ-   d[de(de'dej.d e/d!e/d"e/f d#d$Z0d%d&Z1d'eee2d(e2dej.fd)d*Z3d+eee2d,e2d-ej4fd.d/Z5d0ee%e2ffd1d2Z6de(d-e7fd3d4Z8d5eej9efd6e7d-dfd7d8Z: d\d9ej4d:ee7e7fd;e2d-ej4fdd?Z E d]dFe&dGe%dHe/d-dfdIdJZ?dKdLZ@dMdNZAdOdPZBd-ee%effdQdRZCdggfd5ej9dSe7dTe/dUee%dVee%d-ee'f dWdXZDdS)^N) defaultdict)contextmanager)datetime)Path)AnyDictListTupleUnion)version) distributed)nn)DistributedDataParallel) SummaryWriter GradScalerrc@s4eZdZddZddZddZd defd d Zd S) AttributeDictcCs ||vr||Std|dNzNo such attribute ''AttributeErrorselfkeyr,/home/ubuntu/LuxTTS/zipvoice/utils/common.py __getattr__!szAttributeDict.__getattr__cCs |||<dSNr)rrvaluerrr __setattr__& zAttributeDict.__setattr__cCs"||vr ||=dStd|drrrrrr __delattr__)szAttributeDict.__delattr__indentcCsJi}|D]\}}t|ttjtjfrt|}|||<qtj||ddS)NT)r# sort_keys) items isinstancertorchdevicedtypestrjsondumps)rr#tmpkvrrr__str__/s  zAttributeDict.__str__N)r")__name__ __module__ __qualname__rrr!intr0rrrrr s rcseZdZfddZdddZdeddfdd Zdefd d Zde e eeffd d Z ddZ de dededdfddZZS)MetricsTrackercstt|tdSr)superr5__init__r4r __class__rrr7:szMetricsTracker.__init__otherreturncCsRt}|D]\}}|||<q|D]\}}||dkr&|||||<q|S)Nrr5r%)rr;ansr.r/rrr__add__As  zMetricsTracker.__add__alphacCs(t}|D] \}}||||<q|Srr=)rr@r>r.r/rrr__mul__JszMetricsTracker.__mul__cCsd}d}|D]?\}}d|}d|vr#|t|dt|d7}q|t|dt|7}|dkr8|d7}q|dkrA|d7}qtd |d |d }|d t|d 7}|dkrld |d}|d t|d7}||S)Nz%.4gutt_=z, utt_durationz frames, utt_pad_proportionzUnexpected key: z%.2fframeszover z frames. utterancesz utterances.) norm_itemsr* ValueError)r ans_framesans_utterancesr.r/ norm_valuerGrHrrrr0Ps$    zMetricsTracker.__str__cCsd|vr|dnd}d|vr|dnd}g}|D]$\}}|dks&|dkr'qd|vr1t||nt||}|||fq|S)zb Returns a list of pairs, like: [('ctc_loss', 0.1), ('att_loss', 0.07)] rGrHrC)r%floatappend)r num_framesnum_utterancesr>r.r/rMrrrrIgszMetricsTracker.norm_itemscsbt}tjfdd|D|d}tj|tjjdt|| D]\}}||<q&dS)zs Reduce using torch.distributed, which I believe ensures that all processes get the total. csg|]}t|qSr)rO).0r.r8rr ~sz)MetricsTracker.reduce..r()opN) sortedkeysr'tensordist all_reduceReduceOpSUMzipcputolist)rr(rXsr.r/rr8rreducexs  zMetricsTracker.reduce tb_writerprefix batch_idxNcCs(|D] \}}|||||qdS)a4Add logging information to a TensorBoard writer. Args: tb_writer: a TensorBoard writer prefix: a prefix for the name of the loss, e.g. "train/valid_", or "train/current_" batch_idx: The current batch index, used as the x-axis of the plot. N)rI add_scalar)rrcrdrer.r/rrr write_summaryszMetricsTracker.write_summary)r;r5r<r5)r1r2r3r7r?rOrAr*r0rr rIrbrr4rg __classcell__rrr9rr59s     r5cudac ksttjtdkr-tjjdd|i| dVWddS1s&wYdSt5tjdt dtj jjdi| dVWdn1sQwYWddSWddS1siwYdS)z To fix the following warnings: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead. with torch.cuda.amp.autocast(enabled=False): 2.3.0 device_typeNignorecategoryr) r parser' __version__ampautocastwarningscatch_warnings simplefilter FutureWarningri)rkkwargsrrrtorch_autocasts" "rxcKsttjtdkrddlm}|dd|i|Sttjdt dtj j jdi|WdS1s:wYdS) a Creates a GradScaler compatible with both torch < 2.3.0 and >= 2.3.0. Accepts all kwargs like: enabled, init_scale, growth_factor, etc. FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead. rjrrr(rlrmNr) r ror'rp torch.amprrsrtrurvrirq)r(rwrrrrcreate_grad_scalers  $rzFcCs~dtjvr|dur dnt|tjd<dtjvr$|durdnt|tjd<|dur8tjd||dtj|dStddS) zG rank and world_size are used only if use_ddp_launch is False. MASTER_ADDRN localhost MASTER_PORT12354Fnccl)rank world_size)osenvironr*rZinit_process_groupr'ri set_device)rr master_portuse_ddp_launch master_addrrrr setup_dists  rcCs tdSr)rZdestroy_process_grouprrrr cleanup_distr rTparamsbatchr( return_tokensreturn_feature return_audioc Csdg}|r ||dg7}|r$|d|}|d|}|||j|g7}|r0||d|dg7}|S)a] Parse the features and targets of the current batch. Args: params: It is returned by :func:`get_params`. batch: It is the return value from iterating `lhotse.dataset.K2SpeechRecognitionDataset`. See its documentation for the format of the `batch`. device: The device of Tensor. tokensfeatures features_lensaudio audio_lens)to feat_scale) rrr(rrr return_listrrrrr prepare_inputsrcCsBg}tt|D]}||}|||}||g||q|Sr)rangelenrP)r tokens_lenstokens_durationsirEavg_token_durationrrrprepare_avg_tokens_durationss  rypad_idcsJfdd|D}tdd|Dfdd|D}tj|tj|dS)z Pad the transcripts to the same length with zeros. Args: y: the transcripts, which is a list of a list Returns: Return a Tensor of padded transcripts. csg|]}|gqSrrrS token_ids)rrrrTszpad_labels..cSsg|]}t|qSrrrrrrrTscs"g|] }|gt|qSrrrlengthrrrrTs")r)r()maxr'rYint64)rrr(rrr pad_labelss r durationsrQr<c sfdd|D}t|}tj|tjd}t|D])}||}d}t|D]\}}||||||f<||7}q&|ksCJ|fq|S)aL Gets position in the transcript for each frame, i.e. the position in the symbol-sequence to look up. Args: durations: Duration of each token in transcripts. num_frames: The maximum frame length of the current batch. Returns: Return a Tensor of shape (batch_size, num_frames) csg|] }|t|gqSr)sum)rSxrQrrrT#sz$get_tokens_index..r)r)rr'zerosrr enumerate) rrQ batch_sizer>bthis_dur cur_framerdrrrget_tokens_indexs  rracCs$t|tr|fSttt|dS)N,)r&r4tuplemapsplit)rarrr to_int_tuple0s rcCs|j|j|j|jSr)batch_idx_train max_durationr ref_duration)rrrrget_adjusted_batch_count6s  rmodel batch_countcCsFt|tr|j}|D]\}}t|dr||_t|dr ||_q dS)Nrname)r&DDPmodule named_moduleshasattrrr)rrrrrrrset_batch_count@s   rr mask_percentmax_lencCtj|tjdj||tj}tj|tjd||tj}||}t||}tjd||j d}|dddf|dddfk|dddf|dddfk@}|S)a Apply Time masking. Args: features_lens: input tensor of shape ``(B)`` mask_size: the width size for masking. max_len: the maximum length of the mask. Returns: Return a 2-D bool tensor (B, T), where masked positions are filled with `True` and non-masked positions are filled with `False`. rrrUN) r' zeros_likefloat32uniform_rr rand_likeraranger(rrr mask_size mask_starts mask_ends seq_rangemaskrrrcondition_time_maskKrcCr)a Apply Time masking, mask from the end time index. Args: features_lens: input tensor of shape ``(B)`` mask_size: the width size for masking. max_len: the maximum length of the mask. Returns: Return a 2-D bool tensor (B, T), where masked positions are filled with `True` and non-masked positions are filled with `False`. rrrUN) r'rrrrr ones_likerrr(rrrrcondition_time_mask_suffixnrrlengthscCs^|jdks J|jt||}|d}tjd||jd}|d||}||dkS)a' Args: lengths: A 1-D tensor containing sentence lengths. max_len: The length of masks. Returns: Return a 2-D bool tensor, where masked positions are filled with `True` and non-masked positions are filled with `False`. >>> lengths = torch.tensor([1, 3, 2, 5]) >>> make_pad_mask(lengths) tensor([[False, True, True, True, True], [False, False, False, True, True], [False, False, True, True, True], [False, False, False, False, False]]) rNrrU)ndimrsizer'rr( unsqueezeexpand)rrnrexpaned_lengthsrrr make_pad_masks  rcCs8t|tr|S|dvrdS|dvrdStd)a4Used in argparse.ArgumentParser.add_argument to indicate that a type is a bool type and user can enter - yes, true, t, y, 1, to represent True - no, false, f, n, 0, to represent False See https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse # noqa )yestruetr1T)nofalsefr0FzBoolean value expected.)r&boollowerargparseArgumentTypeError)r/rrrstr2bools    rinfo log_filename log_level use_consolec Cst}|d}tr-tr-t}t}d|d|d}|d|d|}n d}|d|}tj tj |ddt j }|d krLt j}n|d krTt j}n|d kr\t j}n|d krct j}t j|||d dd|rt } | || t |t d| dSdS)aSetup log level. Args: log_filename: The filename to save the log. log_level: The log level to use, e.g., "debug", "info", "warning", "error", "critical" use_console: True to also print logs to console. z%Y-%m-%d-%H-%M-%Sz5%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] (/z ) %(message)s-z?%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)sT)exist_okdebugrwarningcriticalw)filenameformatlevelfilemodeforcerBN)rnowstrftimerZ is_availableis_initializedget_world_sizeget_rankrmakedirspathdirnameloggingERRORDEBUGINFOWARNINGCRITICAL basicConfig StreamHandlersetLevel setFormatter Formatter getLogger addHandler) rrrr date_timerr formatterrconsolerrr setup_loggers@  rcCsz:tjgddtjdjd}ttjgddtjdjddk}|r4|d}W|S|d}W|SYdS) N)git rev-parsez--shortHEADTcheckstdout )rdiffz --shortstatrz-dirtyz-clean) subprocessrunPIPEr decoderstripstripr) git_commit dirty_commitrrr get_git_sha1s:    r+cC:ztjgddtjdjd}W|SYdS)N)rlogz-1z --format=%adz --date=localTrr!r#r$r%r r&r'r(git_daterrr get_git_date r1cCr,)N)rrz --abbrev-refrTrr!r.r/rrrget_git_branch_name'r2r3c CsXttjtjtjjtjddttt tt t  j j tttd S)z Get the environment information.N) z torch-versionztorch-cuda-availableztorch-cuda-versionzpython-versionzzipvoice-git-branchzzipvoice-git-sha1zzipvoice-git-datez zipvoice-pathhostnamez IP address)r*r'rprirr sysr3r+r1r__file__resolveparentsocket gethostname gethostbynamerrrr get_env_info9s  r=lr include_namesfreeze_modulesunfreeze_modulescCst|r t|r Jtdd}g}|D]\}}||t|dr)|j||<qtt} |D]\}} | jsCt d|dq2| d} | d} t|dkrx| dkri| d } | |vrht d|d q2n:| |vrwt d|d q2n+t|dkr| dkr| d } | |vrt d|d q2n| |vrt d|d q2||| }| d kr||d 9}| d d D]}d | |g} ||| 9}q| ||r|| fn| q2|rd d| DSdd| DS)a This is for use with the ScaledAdam optimizers (more recent versions that accept lists of named-parameters; we can, if needed, create a version without the names). It provides a way to specify learning-rate scales inside the module, so that if any nn.Module in the hierarchy has a floating-point parameter 'lr_scale', it will scale the LR of any parameters inside that module or its submodules. Note: you can set module parameters outside the __init__ function, e.g.: >>> a = nn.Linear(10, 10) >>> a.lr_scale = 0.5 Returns: a list of dicts, of the following form: if include_names == False: [ { 'params': [ tensor1, tensor2, ... ], 'lr': 0.01 }, { 'params': [ tensor3, tensor4, ... ], 'lr': 0.005 }, ... ] if include_names == true: [ { 'named_params': [ (name1, tensor1, (name2, tensor2), ... ], 'lr': 0.01 }, { 'named_params': [ (name3, tensor3), (name4, tensor4), ... ], 'lr': 0.005 }, ... ] cSsdS)Ng?rrrrrlsz/get_parameter_groups_with_lrs..lr_scalezRemove z from parameter.rrrNz from parametersrBNcSg|] \}}||dqS)) named_paramsr>r)rSr>pairsrrrrTz1get_parameter_groups_with_lrs..cSrE))rr>r)rSr>rrrrrTrH)rrrrPrrClistnamed_parameters requires_gradr rrjoinr%)rr>r?r@rA flat_lr_scalenamesrm lr_to_params parameter split_namerd module_namecur_lrpartrrrget_parameter_groups_with_lrsIs\         rV)ri)NNNFN)TTF)r)rT)Er collectionsr+r rr:r#r6rsr contextlibrrpathlibrtypingrrrr r r' packagingr r rZr torch.nn.parallelrrtorch.utils.tensorboardrrrqryrtorch.cuda.ampr*Pathlikedictrr5rxrzrrr(rrrr4rTensorrrrOrModulerrrrrrr+r1r3r=rVrrrrs            \    $    &  # 5