o پi$@sdZddlZddlmZmZddlZddlmZddlm Z dej dej d ej d ej d e d e d e de dej de de de de dedededee e fdef$ddZGdddeZdS)a PyTorch MARS Optimizer Code simplified from https://github.com/AGI-Arena/MARS Paper: MARS: Unleashing the Power of Variance Reduction for Training Large Models - https://arxiv.org/abs/2411.10438 @article{yuan2024mars, title={MARS: Unleashing the Power of Variance Reduction for Training Large Models}, author={Yuan, Huizhuo and Liu, Yifeng and Wu, Shuang and Zhou, Xun and Gu, Quanquan}, journal={arXiv preprint arXiv:2411.10438}, year={2024} } N)OptionalTuple) Optimizer)ParamsTpgradexp_avg exp_avg_sqlr weight_decaybeta1beta2 last_gradepsstepgamma mars_type is_grad_2d optimize_1d lr_1d_factorbetas_1dcautioncCs|s| rd|}| dkr|}n||| |||}t|}|dkr*||}||j||d|rO||dk|j}||jdd||}| dkr||j ||d|dd|| }d|| }| t || }|||||}n| d kr||| }nJ|j|| d||fS|\}}||j|d|d||j ||d|dd|| }d|| }| t || }|r||dk|j}||jdd||}|||||}|j||| d||fS) N?r)alphargMbP?)minadamw)valuelion) mul_add_torchnormtodtypediv_meanclamp_addcmul_sqrtmathsign)rrr r r r r rrrrrrrrrrrone_minus_beta1c_tc_t_normmaskbias_correction1bias_correction2denomupdatebeta1_1dbeta2_1dr6C/home/ubuntu/.local/lib/python3.10/site-packages/timm/optim/mars.py_mars_single_tensor_stepsJ     r8cseZdZdZ        dd ed ed eeefdedededededede eeefdeffdd Z fddZ e dddZZS)Marsz MARS Optimizer Paper: MARS: Unleashing the Power of Variance Reduction for Training Large Models https://arxiv.org/abs/2411.10438 ~jth?g?gGz?:0yE>皙?rFrNparamsr betasrr rrrrrrc sd|ks td|d|kstd|d|dkr"dks,ntd|dd|dkr8dksBntd|d|d vsJJd t|||||||| | pV|| d } tt||| dS) Nr=zInvalid learning rate: {}zInvalid epsilon value: {}rrz%Invalid beta parameter at index 0: {}rz%Invalid beta parameter at index 1: {})rrzMARS type not supported) r r@rr rrrrrr) ValueErrorformatdictsuperr9__init__) selfr?r r@rr rrrrrrdefaults __class__r6r7rEbs, z Mars.__init__cs,tt|||jD]}|ddq dS)NrF)rDr9 __setstate__ param_groups setdefault)rFstategrouprHr6r7rJs zMars.__setstate__cCsfd}|durt |}Wdn1swY|jD]}|dD]}|jdur.q&|j}|jr8td|j|}t|dkr\d|d<t||d<t||d<t||d <|dd7<|d}|d}|d } |d} |d } |d } |d \} }|j d k}t |||| | | | || |d||d|d||d|d|d|dd||d<q&q |S)zPerforms a single optimization step. Arguments: closure (callable, optional): A closure that reevaluates the model and returns the loss. Nr?zJAdam does not support sparse gradients, please consider SparseAdam insteadrrrr rr r r r@rrrrrrr)rrrrrr) r! enable_gradrKr is_sparse RuntimeErrorrMlen zeros_likendimr8)rFclosurelossrNrrrMrr r rr wdr rrr6r6r7rsd         4z Mars.step) r:r;r<r=r>rFrNF)N)__name__ __module__ __qualname____doc__rfloatrstrboolrrErJr!no_gradr __classcell__r6r6rHr7r9[sL       &r9)r\r*typingrrr!torch.optim.optimizerr_typesrTensorr]intr^r_r8r9r6r6r6r7sX         B