o ,i<"@sddlmZmZmZmZddlZddlmZddlmZm Z m Z m Z m Z m Z mZmZmZmZmZmZmZddgZGdddeZd e d ed e d ed e_d eed eedeedeedeedeedededededededededefddZd eed eedeedeedeedeedededededededededefddZe ed  d#d eed eedeedeedeedeed!eedededededededededef d"dZdS)$)ListOptionalTupleUnionN)Tensor) _capturable_doc_default_to_fused_or_foreach_differentiable_doc_disable_dynamo_if_unsupported _foreach_doc!_get_capturable_supported_devices_get_scalar_dtype _get_value _maximize_doc_use_grad_for_differentiable _view_as_real OptimizerParamsTASGDasgdcseZdZ         dded ed ed ed ed edeedededeffdd ZfddZddZ e dddZ Z S)r{Gz?-C6??.ArNFparamslrlambdalphat0 weight_decayforeachmaximizedifferentiable capturablec sXd|ks td|d|kstd|t|||||||| | d } t|| dS)NgzInvalid learning rate: zInvalid weight_decay value: ) rrrrr r!r"r#r$) ValueErrordictsuper__init__) selfrrrrrr r!r"r#r$defaults __class__N/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/optim/asgd.pyr(s   z ASGD.__init__cst||jD]q}|dd|dd|dd|dd|dD]R}|j|g}t|dkryt|dsOt |d}tj |t |j d |d<t|d sdtj |d t |j d |d <t|d sytj |d t |j d |d <q'q dS) Nr!r"Fr#r$rrstep)dtypedeviceetamu) r' __setstate__ param_groups setdefaultstategetlentorch is_tensorfloattensorrr1)r)r7grouppp_statestep_valr+r-r.r4:s2             zASGD.__setstate__c Cs d}|dD]|} | jdur|t| O}|| | jjr!td|| j|j| } t| dkrftjd| j t d| d<tj |d| j t d | d <tjd| j t d| d <tj| tjd | d <|| d || d || d || dq|S) NFrz&ASGD does not support sparse gradientsrr-)r1r0r/rr2r3) memory_formatax)gradr: is_complexappend is_sparse RuntimeErrorr7r9zerosr1r as_tensorclonedetachones zeros_likepreserve_format) r)r>params_with_gradgradsmusaxsetas state_steps has_complexr?r7r-r-r. _init_groupRs>           zASGD._init_groupc Cs|d}|dur!t |}Wdn1swY|jD]?}g}g}g}g}g}g} |||||||| } t|||||| |d|d|d|d|d|d|d|d |d | d q$|S) zPerform a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nrrrrr r!r"r#r$) rrrrr r!r"r#r$rV) _cuda_graph_capture_health_checkr: enable_gradr5rWr) r)closurelossr>rPrQrRrSrTrUrVr-r-r.r/vsF  z ASGD.step) rrrrrNFFFN) __name__ __module__ __qualname__rr<rboolr(r4rWrr/ __classcell__r-r-r+r.rsF    $ahImplements Averaged Stochastic Gradient Descent. It has been proposed in `Acceleration of stochastic approximation by averaging`_. Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-2) lambd (float, optional): decay term (default: 1e-4) alpha (float, optional): power for eta update (default: 0.75) t0 (float, optional): point at which to start averaging (default: 1e6) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) z zx .. _Acceleration of stochastic approximation by averaging: https://dl.acm.org/citation.cfm?id=131098 rrQrSrRrTrUrrrrr r"r#r$rVc  Cst|D]\}}||}| s|n| }||}||}||}||}tjsU| rUt}|jj|jjkrE|jjkrE|jjkrMnn|jj|vsUJd|dt|rit|}t|}t|}|d7}| dkrx|j || d}| r| d|||j ||ddnt |}| d|||j || d| s|dkr| || |n||| r||d|||| |dt||t|qt |}t|d|||| }||tdtd||}||qdS)NUIf capturable=True, params, mus, etas, and state_steps must be on supported devices: .rrrvalue) enumerater:_utils is_compilingr r1typerE view_as_realaddmul_addcmul_radd_itemsubcopy_maximum ones_likerJmax)rrQrSrRrTrUrrrrr r"r#r$rViparamrDr3rCr2step_tcapturable_supported_devices eta_valuer/new_etanew_mur-r-r._single_tensor_asgds\      "  r~c sDt|dkrdS| rJdtjs2| r2tddtfddt||||Ds2Jddt||||||g}| D]\\}\\}}}}}}}|rWt |||| r^t |}|dj rqtj |tjd d d d d nt |d | dkr| rtj ||| d |}ntj||| d }tj ||d ntj||d }tj|||dd~t||}t|||~| rt|}t|d t|t||~t|}t|t |d t|t|t|t||qAfdd|D}fdd|D}t||t||qAdS)Nrz#_foreach ops don't support autogradF) supports_xlac3sV|]&\}}}}|jj|jjko|jjko|jjkno&|jjvVqdSr\)r1rk).0r?r3r2r/)rzr-r. 's  2  z%_multi_tensor_asgd..rbrcg?cpur1rdrrerfcs.g|]}tjd|dqSrr)r:rJrr/)rr1rrr-r. s z&_multi_tensor_asgd..cs,g|]}tjdtdt|dqSr)r:rJrvrr)r1rr-r.rs)r9r:rirjr allzipr"_group_tensors_by_device_and_dtypeitemsr _foreach_negis_cpu _foreach_add_r= _foreach_add_foreach_addcmul_ _foreach_sub_foreach_maximum__foreach_reciprocal__foreach_copy_ _foreach_mul _foreach_mul_ _foreach_pow_)rrQrSrRrTrUrrrrr r"r#r$rVgrouped_tensors_grouped_params grouped_grads grouped_axs grouped_mus grouped_etasgrouped_state_steps intermediatenew_musnew_etasr-)rrzr1rrrr._multi_tensor_asgd s                       r)single_tensor_fnFr!c Csr|dur t||dd\}}|rtjrtd|r"tjs"t}nt}|||||||| | | ||||| | ddS)znFunctional API that performs asgd algorithm computation. See :class:`~torch.optim.ASGD` for details. NF) use_fusedz6torch.jit.script not supported with foreach optimizers) rrrrr r"r#r$rV)r r:jit is_scriptingrHrr~)rrQrSrRrTrUr!r"r#r$rVrrrrr rfuncr-r-r.rs4  )NFFFF)typingrrrrr:r optimizerrr r r r r rrrrrrr__all__r__doc__r<r`r~rrr-r-r-r.s <        L