o
    پif                     @   sZ   d Z ddlZddlZddlZddlZddlmZ ddlm	Z	 e
eZG dd de	ZdS )z Cosine Scheduler

Cosine LR schedule with warmup, cycle/restarts, noise, k-decay.

Hacked together by / Copyright 2021 Ross Wightman
    N)List   )	Schedulerc                       s   e Zd ZdZ													
		ddejjdedededededdf fddZ	dede
e fddZdddZ  ZS )CosineLRSchedulera\  
    Cosine decay with restarts.
    This is described in the paper https://arxiv.org/abs/1608.03983.

    Inspiration from
    https://github.com/allenai/allennlp/blob/master/allennlp/training/learning_rate_schedulers/cosine.py

    k-decay option based on `k-decay: A New Method For Learning Rate Schedule` - https://arxiv.org/abs/2004.05909
                  ?r   r   FTNq=
ףp?*   	optimizer	t_initiallr_min	cycle_mulcycle_decaycycle_limitreturnc              
      s   t  j|d|
|||||d |dksJ |dksJ |dkr+|dkr+|dkr+td | _| _| _| _| _| _	 _
|	 _| _ j	r^ fdd jD  _t   j
 d S dd  jD  _d S )	Nlr)param_group_fieldt_in_epochsnoise_range_t	noise_pct	noise_std
noise_seed
initializer   r   zjCosine annealing scheduler will have no effect on the learning rate since t_initial = t_mul = eta_mul = 1.c                    s   g | ]	}|  j  qS  )warmup_t.0vselfwarmup_lr_initr   L/home/ubuntu/.local/lib/python3.10/site-packages/timm/scheduler/cosine_lr.py
<listcomp>L       z.CosineLRScheduler.__init__.<locals>.<listcomp>c                 S   s   g | ]}d qS )r   r   r   _r   r   r!   r"   O   s    )super__init___loggerwarningr   r   r   r   r   r   r    warmup_prefixk_decaybase_valueswarmup_stepsupdate_groups)r   r
   r   r   r   r   r   r   r    r*   r   r   r   r   r   r+   r   	__class__r   r!   r'      s:   zCosineLRScheduler.__init__tc                    s  j k rfddjD }|S jrj  jdkrMttdj dj   j}j| j dj|  dj  j  nj }jj|  j|   fddj	D }j
|jk rfdd|D }|S fddj	D }|S )Nc                    s   g | ]	} j |  qS r   )r    )r   s)r   r1   r   r!   r"   S   r#   z-CosineLRScheduler._get_lr.<locals>.<listcomp>r   c                    s   g | ]}|  qS r   r   r   )gammar   r!   r"   b   s    c              
      s@   g | ]}j d |j   dttj         qS )g      ?r   )r   mathcospi)r   lr_max)kr   t_currt_ir   r!   r"   f   s    2c                    s   g | ]} j qS r   )r   r$   )r   r   r!   r"   k   s    )r   r-   r*   r   r4   floorlogr   r   r,   r+   r   )r   r1   lrsilr_max_valuesr   )r3   r8   r   r1   r9   r:   r!   _get_lrQ   s,   


($


zCosineLRScheduler._get_lrc                 C   sd   t d|p| j}| jdkr| j| }ntt| j | j| d  d| j  }| jr0|| j S |S )Nr   r   )	maxr   r   r   intr4   r;   r*   r   )r   cyclesr1   r   r   r!   get_cycle_lengtho   s
   
*z"CosineLRScheduler.get_cycle_length)r   r   r   r   r   r   FTNr   r   r	   r   T)r   )__name__
__module____qualname____doc__torchoptim	OptimizerrB   floatr'   r   r@   rD   __classcell__r   r   r/   r!   r      sB    3r   )rH   loggingr4   numpynprI   typingr   	schedulerr   	getLoggerrE   r(   r   r   r   r   r!   <module>   s    
