o ߗi@svUddlmZmZmZmZddlZddlmmZ ddlm Z ddl m Z gZ eeed<ejjGdddZdS))DictListOptionalTupleN)Tensor)2_scripted_functional_optimizer_deprecation_warning__all__c@seZdZ         ddeededeeefd ed ed ed ed ededefddZdede efddZ dee efddZ dS)_FunctionalAdamWMbP?g?g+?:0yE>{Gz?Fparamslrbetaseps weight_decayamsgradmaximizeforeachfused_allow_empty_param_listc Cs tddd|kstd|d|kstd|d|dkr'dks1ntd|dd|d kr=dksGntd |d d|ksRtd ||||d|d |d |_||_||_||_| |_tj t tj t t tj ffi|_ t|dkr| std d|i|_dS)N) stacklevelzInvalid learning rate: zInvalid epsilon value: rg?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: )rrbeta1beta2rz%optimizer got an empty parameter listr)r ValueErrordefaultsrrrrtorchjitannotaterrstrstatelen param_group) selfrrrrrrrrrrr(f/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/torch/distributed/optim/functional_adamw.py__init__s2 $z_FunctionalAdamW.__init__paramgradc Csvg}g}g}g}g}g}t|} |dur||||||jvrVi|j|<|j|} td| d<tj|tjd| d<tj|tjd| d<|jrVtj|tjd| d<|j|} || d|| d|jrs|| d|| dt3t j |||||||j|j |j d|j d|j d |j d |j d |j |jdd| d WddS1swYdS) Nrstep memory_formatexp_avg exp_avg_sqmax_exp_avg_sqrrrrr rrrrrrrrr grad_scale found_inf has_complex)r is_complexappendr$tensor zeros_likepreserve_formatrno_gradFadamwrrrr) r'r+r,params_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr6r$r(r(r) step_paramGsh          "z_FunctionalAdamW.step_param gradientsc Cs|jd}g}g}g}g}g}g}t|t|kr,tddt|ddt|d} t|jd|D]t\} } | dur| t| O} || || | |jvri|j| <|j| } td| d<tj | tj d | d <tj | tj d | d <|j rtj | tj d | d <|j| } || d || d |j r|| d || dq6t 3t j|||||||j |j|jd |jd|jd|jd|jd|j|jdd| dWddS1swYdS)NrzEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: Frr-r.r0r1r2rrrrrr3)r&r%rzipr r7r8r$r9r:r;rr<r=r>rrrr) r'rFrr?r@rArBrCrDr6r+gradientr$r(r(r)r-s           "z_FunctionalAdamW.stepN) r r r r FFFFF) __name__ __module__ __qualname__rrfloatrboolr*rrEr-r(r(r(r)r sB    -=r )typingrrrrr torch.optim._functionaloptim _functionalr=r,torch.distributed.optim._deprecation_warningrrr#__annotations__r!scriptr r(r(r(r)s