o }oi3<"@sddlZddlmZddlZddlmZddlmZGdddeZGdddeZ d eed eed eed eed eedeede de de de de de de de de de def"ddZ d eed eed eed eed eedeede de de de de de de de de de def"ddZ d eed eed eed eed eedeede de de de de de de de de de def"ddZd eed eed eed eed eedeede de de de de de de de de de def"d d!ZdS)"N)List)Tensor) Optimizerc@s$eZdZdZdZddZddZdS)MultiTensorApplyFc CsFz dt_||_WdSty"}z dt_|t_WYd}~dSd}~ww)NTF)r available chunk_size ImportError import_err)selfrerrr H/home/ubuntu/.local/lib/python3.10/site-packages/nemo/core/optim/adan.py__init__)s zMultiTensorApply.__init__cGs||j||g|RSN)r)r opnoop_flag_buffer tensor_listsargsr r r __call__1szMultiTensorApply.__call__N)__name__ __module__ __qualname__rwarnedrrr r r r r%s  rcsjeZdZdZ        dded effd d Zfd d ZeddZ edddZ Z S)Adana Implements a pytorch variant of Adan Adan was proposed in Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models[J].arXiv preprint arXiv:2208.06677, 2022. https://arxiv.org/abs/2208.06677 Arguments: params (iterable): iterable of parameters to optimize or dicts defining parameter groups. lr (float, optional): learning rate. (default: 1e-3) betas (Tuple[float, float, flot], optional): coefficients used for first- and second-order moments. (default: (0.98, 0.92, 0.99)) eps (float, optional): term added to the denominator to improve numerical stability. (default: 1e-8) weight_decay (float, optional): decoupled weight decay (L2 penalty) (default: 0) max_grad_norm (float, optional): value used to clip global grad norm (default: 0.0 no clip) no_prox (bool): how to perform the decoupled weight decay (default: False) foreach (bool): if True would use torch._foreach implementation. It's faster but uses slightly more memory. (default: True) fused (bool, optional): whether fused implementation is used. (default: False) MbP?g\(\?gq= ףp?gGz?:0yE>FTforeachfusedc sd|ks td|d|kstd|d|ks!td|d|dkr-dks7ntd|dd|dkrCdksMntd |dd|d krYdkscntd |d t|||||||| d } t|| dS) NrzInvalid Max grad norm: {}zInvalid learning rate: {}zInvalid epsilon value: {}r?z%Invalid beta parameter at index 0: {}z%Invalid beta parameter at index 1: {}z%Invalid beta parameter at index 2: {})lrbetaseps weight_decay max_grad_normno_proxrr) ValueErrorformatdictsuperr) r paramsr#r$r%r&r'r(rrdefaults __class__r r rPs.  z Adan.__init__cs,tt|||jD]}|ddq dS)Nr(F)r,r __setstate__ param_groups setdefault)r stategroupr/r r r1ts zAdan.__setstate__cCs`|jD]*}d|d<|dD]}|jr,|j|}t||d<t||d<t||d<q qdS)Nrstepr-exp_avg exp_avg_sq exp_avg_diff)r2 requires_gradr4torch zeros_like)r r5pr4r r r restart_optys   zAdan.restart_optNcCsRd}|durt |}Wdn1swY|jddkrs|jdddj}tjd|d}tj|jd|d}|jD]}|dD]}|jdur\|j}|| d qHqBt |}tj |||dd d  } nd } |jD]-}g} g} g} g} g}g}|d \}}}d |vr|d d7<nd|d <d ||d }d ||d }d ||d }|dD]e}|jdurq| || |j|j|}t|dkrt||d <t||d<t||d<d|vs|d dkr|j| |d<| |d | |d||d||dqtd#id| d| d| d| d|d|d|d|d|d|d|dt |d|dd|dd|dd|dd| }|d r|d!rtjr|td#i|qxtd"td#i|qx|d!rtjrtd#i|qxtd"td#i|qx|S)$z$Performs a single optimization step.Nr'rr-r!)devicer"r%r )maxr$r6r7r8r9 neg_pre_gradgradsexp_avgs exp_avg_sqs exp_avg_diffs neg_pre_gradsbeta1beta2beta3bias_correction1bias_correction2bias_correction3_sqrtr#r&r(clip_global_grad_normrrzFused Adan does not support CPUr )r; enable_gradr.r2r?zerostensorgradadd_powsumsqrtclampitemappendr4lenr<clonemul_r+mathcuda is_available_fused_adan_multi_tensorr)_multi_tensor_adan_fused_adan_single_tensor_single_tensor_adan)r closurelossr?global_grad_normr'r5r=rQrMparams_with_gradrBrCrDrErFrGrHrIrJrKbias_correction3r4kwargsr r r r6s                       z Adan.step)rrrrrFTFr) rrr__doc__boolrr1r;no_gradr>r6 __classcell__r r r/r r5s(   $ rr-rBrCrDrErFrGrHrIrJrKrLr#r&r%r(rMc CsVt|D]\}}||}||}||}||}||}||||||j|d|d||j|d|d|||||j||d|d|| |}| || }| | }|r|d| | |j||| d|j||| dn|j||| d|j||| d|d| | |j|ddqdS)Nr!alphavalue) enumerater[rRaddcmul_rUaddcdiv_div_zero_)r-rBrCrDrErFrGrHrIrJrKrLr#r&r%r(rMiparamrQr7r8r9neg_grad_or_diffdenomstep_size_diff step_sizer r r rbs0   rbc Cstt|dkrdSt||t||t||tj||d|dt||tj||d|dt||t||t||tj|||d|dt|}t|| t||| || }| | }|rt|d| | tj|||| dtj|||| dntj|||| dtj|||| dt|d| | t|tj||dddS)Nrr!rmrorq) rYr; _foreach_mul_ _foreach_add__foreach_addcmul_ _foreach_sqrt _foreach_div__foreach_addcdiv__foreach_zero_)r-rBrCrDrErFrGrHrIrJrKrLr#r&r%r(rMrzr{r|r r r r`+s4             r`c Cslddl}td}tjdg}||j|||||||g|||| | | | | |||t|tj||dddS)Nrirqrm) fused_adanrr;r] IntTensoradan_multi_tensorrr~)r-rBrCrDrErFrGrHrIrJrKrLr#r&r%r(rMrmulti_tensor_applier_dummy_overflow_bufr r r r_fs* r_c Cst|D]Z\}}|j}|j}||}||}||}||}||}tj|j"ddl}|||||||||||| | | | | |||Wdn1sPwY|j |ddqdS)Nrrqrm) rrdatafloatr;r]r?radan_single_tensorrvrR)r-rBrCrDrErFrGrHrIrJrKrLr#r&r%r(rMrwrx p_data_fp32out_prQr7r8r9neg_gradrr r r rasB ra)r\typingrr;rtorch.optim.optimizerrobjectrrrrjrbr`r_rar r r r s(   @      7      ;      ,