o
    oi#                     @   sL   d dl Z d dlZd dlmZmZ ddlmZmZmZm	Z	 G dd deZ
dS )    N)	Optimizerrequired   )Betas2OptFloatOptLossClosureParamsc                       s|   e Zd ZdZ							dd	ed
ededededededef fddZde	ddf fddZ
ddedefddZ  ZS )Rangera   Implements Ranger optimization algorithm

    Args:
        params: iterable of parameters to optimize or dicts defining
            parameter groups
        lr: learning rate (default: 1e-3)
        alpha: linear interpolation factor. 1.0 recovers the inner optimizer.
            (default: 0.5)
        k: number of lookahead steps (default: 6)
        N_sma_threshhold: Maximum length of the simple moving average (SMA)
        betas: coefficients used for computing
            running averages of gradient and its square (default: (0.95, 0))
        eps: term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay: weight decay (L2 penalty) (default: 0)

    Example:
        >>> from pytorch_ranger import Ranger
        >>> optimizer =  Ranger(model.parameters(), lr=0.1)
        >>> optimizer.zero_grad()
        >>> loss_fn(model(input), target).backward()
        >>> scheduler = StepLR(optimizer, step_size=1, gamma=0.7)
        >>> optimizer.step()
        >>> scheduler.step()
    MbP?      ?      gffffff?g+?h㈵>r   paramslralphakN_sma_threshholdbetasepsweight_decayc	           
   
      s   d|  kr
dksn t d|d|kst d||dks(t d||dks3t d|t|||d||||d	}	t ||	 || _|| _|| _d
d tdD | _	d S )Ng              ?zInvalid slow update rate: {}r   zInvalid lookahead steps: {}r   zInvalid Learning Rate: {}zInvalid eps: {})r   r   r   step_counterr   r   r   r   c                 S   s   g | ]}g d qS ))NNN ).0indr   r   S/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/pytorch_ranger/ranger.py
<listcomp>g   s    z#Ranger.__init__.<locals>.<listcomp>
   )

ValueErrorformatdictsuper__init__r   r   r   rangeradam_buffer)
selfr   r   r   r   r   r   r   r   defaults	__class__r   r   r$   7   s"   	
zRanger.__init__statereturnNc                    s   t  | d S N)r#   __setstate__)r'   r+   r)   r   r   r.   v   s   zRanger.__setstate__closurec                 C   s  d}| j D ]k}|d D ]c}|jdu rq|jj }|jr"td|j }| j| }t|dkrUd|d< t	||d< t	||d< t
|j|d< |d |j n|d ||d< |d ||d< |d |d }}	|d	 \}
}|	|d
| || ||
d
|
 | |d  d
7  < | jt|d d  }|d |d kr|d
 |d }}n^|d |d< ||d  }dd
|  d
 }|d|d  | d
|   }||d
< || jkrtd
| |d  |d  |d  | | |d  d
|
|d    }n
dd
|
|d    }||d< |d dkr$||d  |d  | || jkr@|	 |d }|| |d  || n|| |d  | |j| |d |d  dkrp|d }|| j|j|  |j| qq|S )zPerforms a single optimization step.

        Arguments:
            closure: A closure that reevaluates the model and returns the loss.
        Nr   z2Ranger optimizer does not support sparse gradientsr   stepexp_avg
exp_avg_sqslow_bufferr   r   r         r   r   r   r   r   )param_groupsgraddatafloat	is_sparseRuntimeErrorr+   lentorch
zeros_like
empty_likecopy_type_asmul_addcmul_add_r&   intr   mathsqrtaddcdiv_r   )r'   r/   lossgrouppr7   p_data_fp32r+   r1   r2   beta1beta2bufferedN_sma	step_sizebeta2_t	N_sma_maxdenomslow_pr   r   r   r0   y   s   



OzRanger.step)r
   r   r   r   r   r   r   r-   )__name__
__module____qualname____doc__r   r9   rE   r   r$   r"   r.   r   r   r0   __classcell__r   r   r)   r   r	      s8    	?r	   )rF   r=   torch.optim.optimizerr   r   typesr   r   r   r   r	   r   r   r   r   <module>   s
   