o
    yiK!                     @   s4   d dl Z d dlZd dlmZmZ G dd deZdS )    N)	Optimizerrequiredc                       s:   e Zd Z			d fdd	Z fddZdddZ  ZS )RangerVAMbP?      ?      gffffff?g+?h㈵>r   Tsoftplus2   squarec                    s   d|  kr
dksn t d|d|kst d||dks(t d||dks3t d|t|||d||||||
||	d	}t || || _|| _|| _d
d tdD | _	d S )Ng        g      ?zInvalid slow update rate: {}   zInvalid lookahead steps: {}r   zInvalid Learning Rate: {}zInvalid eps: {})lralphakstep_counterbetasn_sma_threshholdepsweight_decaysmoothtransformergrad_transformeramsgradc                 S   s   g | ]}g d qS ))NNN ).0indr   r   M/home/ubuntu/.local/lib/python3.10/site-packages/pytorch_ranger/ranger913A.py
<listcomp>;   s    z%RangerVA.__init__.<locals>.<listcomp>
   )

ValueErrorformatdictsuper__init__r   r   r   rangeradam_buffer)selfparamsr   r   r   r   r   r   r   r   r   r   r   defaults	__class__r   r   r%      s$   zRangerVA.__init__c                    s   t  | d S N)r$   __setstate__)r(   stater+   r   r   r.   J   s   zRangerVA.__setstate__Nc                 C   s  d }| j D ]f}|d D ]^}|jd u rq|jj }|jr"td|d }|d }|d }|j }	| j| }
t|
dkrkd|
d< t	|	|
d< t	|	|
d	< |rZt	|j|
d
< t
|j|
d< |
d |j n|
d |	|
d< |
d	 |	|
d	< |
d |
d	 }}|d \}}|r|
d
 }||d| || ||d| | |dkr|d }n|dkr| }||d| |  |rtj|||d | }n| }|dkr|  |
d  d7  < |d dkr|	|d  |d  |	 d||
d   }d||
d   }|d t| | }|d dkr1tj|}||}|	| || n| |d }|	| |d  || |j|	 |
d |d  dkrk|
d }|| j|j|  |j| qq|S )Nr)   z2Ranger optimizer does not support sparse gradientsr   r   r   r   stepexp_avg
exp_avg_sqmax_exp_avg_sqslow_bufferr   r   r      abs)outr   r   r   r   r   r   )param_groupsgraddatafloat	is_sparseRuntimeErrorr/   lentorch
zeros_like
empty_likecopy_type_asmul_addcmul_add_r6   maxclonesqrt_mathsqrtnnSoftplusaddcdiv_r   )r(   closurelossgrouppr9   r   r   r   p_data_fp32r/   r1   r2   beta1beta2r3   grad_tmpdenomcbias_correction1bias_correction2	step_sizespdenomfdenomslow_pr   r   r   r0   N   sx   




qzRangerVA.step)r   r   r   r   r	   r
   r   Tr   r   r   r-   )__name__
__module____qualname__r%   r.   r0   __classcell__r   r   r+   r   r      s    0r   )rJ   r?   torch.optim.optimizerr   r   r   r   r   r   r   <module>   s   