o
    }oi9                     @   s  d dl mZ d dlmZmZmZmZmZmZm	Z	m
Z
mZmZmZ d dlmZ G dd deZG dd deZG dd	 d	eZG d
d deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZdS )    )Optional)InverseSquareRootAnnealingNoamAnnealingNoamHoldAnnealingPolynomialDecayAnnealingPolynomialHoldDecayAnnealingSquareAnnealingSquareRootAnnealingT5InverseSquareRootAnnealingWarmupAnnealingWarmupHoldPolicyWarmupPolicy)LRSchedulerModulec                       Z   e Zd ZdZ							dd	ed
ee dededededef fddZdd Z	  Z
S )WarmupPolicySchedulerz&Warmup Policy Learning Rate Scheduler.  N
           step   val_losswarmup_stepswarmup_ratio	max_stepsmin_lrinterval	frequencymonitorc                    8   t    || _|| _|| _|| _|| _|| _|| _d S N	super__init__r   r   r   r   r   r   r   selfr   r   r   r   r   r   r   	__class__ ]/home/ubuntu/.local/lib/python3.10/site-packages/nemo/lightning/pytorch/optim/lr_scheduler.pyr"   $      


zWarmupPolicyScheduler.__init__c                 C   4   t || j| j| j| jd}||| j| jd| jdS N)r   r   r   r   	schedulerr   r   	optimizerlr_schedulerr   )r   r   r   r   r   r   r   r   r$   modelr/   r0   r'   r'   r(   r-   7      zWarmupPolicyScheduler.schedulerr   Nr   r   r   r   r   __name__
__module____qualname____doc__intr   floatstrr"   r-   __classcell__r'   r'   r%   r(   r   !   2    r   c                       sn   e Zd ZdZ									dd	ed
ee dee dee dededededef fddZdd Z	  Z
S )WarmupHoldPolicySchedulerz+Warmup Hold Policy Learning Rate Scheduler.r   Nr   r   r   r   r   r   r   
hold_steps
hold_ratior   r   r   r   r   c
           
         sD   t    || _|| _|| _|| _|| _|| _|| _|| _	|	| _
d S r   )r!   r"   r   r   r@   rA   r   r   r   r   r   )
r$   r   r   r@   rA   r   r   r   r   r   r%   r'   r(   r"   M   s   

z"WarmupHoldPolicyScheduler.__init__c              	   C   s<   t || j| j| j| j| j| jd}||| j| jd| j	dS )N)r   r   r@   rA   r   r   r,   r.   )
r   r   r   r@   rA   r   r   r   r   r   r1   r'   r'   r(   r-   d   s    
z#WarmupHoldPolicyScheduler.scheduler)	r   NNNr   r   r   r   r   r5   r'   r'   r%   r(   r?   J   s>    	
r?   c                       J   e Zd ZdZ					ddeded	ed
edef
 fddZdd Z  Z	S )SquareAnnealingSchedulerz)Square Annealing Learning Rate Scheduler.r   h㈵>r   r   r   r   r   r   r   r   c                    ,   t    || _|| _|| _|| _|| _d S r   r!   r"   r   r   r   r   r   r$   r   r   r   r   r   r%   r'   r(   r"   |      

z!SquareAnnealingScheduler.__init__c                 C   ,   t || j| jd}||| j| jd| jdS N)r   r   r,   r.   )r   r   r   r   r   r   r1   r'   r'   r(   r-         z"SquareAnnealingScheduler.scheduler)r   rD   r   r   r   
r6   r7   r8   r9   r:   r;   r<   r"   r-   r=   r'   r'   r%   r(   rC   y   &    rC   c                       rB   )SquareRootAnnealingSchedulerz.Square Root Annealing Learning Rate Scheduler.r   r   r   r   r   r   r   r   r   r   c                    rE   r   rF   rG   r%   r'   r(   r"      rH   z%SquareRootAnnealingScheduler.__init__c                 C   rI   rJ   )r	   r   r   r   r   r   r1   r'   r'   r(   r-      rK   z&SquareRootAnnealingScheduler.schedulerr   r   r   r   r   rL   r'   r'   r%   r(   rN      rM   rN   c                       s^   e Zd ZdZ							dd	ed
edee dededededef fddZdd Z	  Z
S )NoamAnnealingSchedulerz'Noam Annealing Learning Rate Scheduler.r   Nr   r   r   r   r   d_modelr   r   r   r   r   r   r   c	           	         s>   t    || _|| _|| _|| _|| _|| _|| _|| _	d S r   )
r!   r"   rQ   r   r   r   r   r   r   r   )	r$   rQ   r   r   r   r   r   r   r   r%   r'   r(   r"      s   

zNoamAnnealingScheduler.__init__c                 C   s8   t || j| j| j| j| jd}||| j| jd| jdS )N)rQ   r   r   r   r   r,   r.   )	r   rQ   r   r   r   r   r   r   r   r1   r'   r'   r(   r-      s   	z NoamAnnealingScheduler.schedulerr4   r5   r'   r'   r%   r(   rP      s6    	rP   c                       sP   e Zd ZdZ						dded	ed
edededef fddZdd Z  Z	S )NoamHoldAnnealingSchedulerz,Noam Hold Annealing Learning Rate Scheduler.r         ?r   r   r   r   r   
decay_rater   r   r   r   c                    s2   t    || _|| _|| _|| _|| _|| _d S r   )r!   r"   r   rT   r   r   r   r   )r$   r   rT   r   r   r   r   r%   r'   r(   r"      s   
	
z#NoamHoldAnnealingScheduler.__init__c                 C   s0   t || j| j| jd}||| j| jd| jdS )N)r   rT   r   r,   r.   )r   r   rT   r   r   r   r   r1   r'   r'   r(   r-      s   z$NoamHoldAnnealingScheduler.scheduler)r   rS   r   r   r   r   rL   r'   r'   r%   r(   rR      s,    rR   c                       r   )WarmupAnnealingSchedulerz)Warmup Annealing Learning Rate Scheduler.r   Nr   r   r   r   r   r   r   r   r   r   r   r   c                    r   r   r    r#   r%   r'   r(   r"   	  r)   z!WarmupAnnealingScheduler.__init__c                 C   r*   r+   )r   r   r   r   r   r   r   r   r1   r'   r'   r(   r-     r3   z"WarmupAnnealingScheduler.schedulerr4   r5   r'   r'   r%   r(   rU     r>   rU   c                       rB   )#InverseSquareRootAnnealingSchedulerz6Inverse Square Root Annealing Learning Rate Scheduler.r   r   r   r   r   r   r   r   r   r   c                    rE   r   rF   rG   r%   r'   r(   r"   2  rH   z,InverseSquareRootAnnealingScheduler.__init__c                 C   rI   rJ   )r   r   r   r   r   r   r1   r'   r'   r(   r-   A  rK   z-InverseSquareRootAnnealingScheduler.schedulerrO   rL   r'   r'   r%   r(   rV   /  rM   rV   c                       rB   )%T5InverseSquareRootAnnealingSchedulerz9T5 Inverse Square Root Annealing Learning Rate Scheduler.r   r   r   r   r   r   r   r   r   r   c                    rE   r   rF   rG   r%   r'   r(   r"   Q  rH   z.T5InverseSquareRootAnnealingScheduler.__init__c                 C   rI   rJ   )r
   r   r   r   r   r   r1   r'   r'   r(   r-   `  rK   z/T5InverseSquareRootAnnealingScheduler.schedulerrO   rL   r'   r'   r%   r(   rW   N  rM   rW   c                       V   e Zd ZdZ							dd	ed
edededededef fddZdd Z	  Z
S )!PolynomialDecayAnnealingSchedulerz3Polynomial Decay Annealing Learning Rate Scheduler.r   r         ?Fr   r   r   r   r   powercycler   r   r   c                    r   r   	r!   r"   r   r   r[   r\   r   r   r   r$   r   r   r[   r\   r   r   r   r%   r'   r(   r"   p  r)   z*PolynomialDecayAnnealingScheduler.__init__c                 C   r*   N)r   r   r[   r\   r,   r.   )r   r   r   r[   r\   r   r   r   r1   r'   r'   r(   r-        z+PolynomialDecayAnnealingScheduler.schedulerr   r   rZ   Fr   r   r   r6   r7   r8   r9   r:   r;   boolr<   r"   r-   r=   r'   r'   r%   r(   rY   m  2    rY   c                       rX   )%PolynomialHoldDecayAnnealingSchedulerz8Polynomial Hold Decay Annealing Learning Rate Scheduler.r   r   rZ   Fr   r   r   r   r   r[   r\   r   r   r   c                    r   r   r]   r^   r%   r'   r(   r"     r)   z.PolynomialHoldDecayAnnealingScheduler.__init__c                 C   r*   r_   )r   r   r   r[   r\   r   r   r   r1   r'   r'   r(   r-     r`   z/PolynomialHoldDecayAnnealingScheduler.schedulerra   rb   r'   r'   r%   r(   re     rd   re   c                       sR   e Zd Z							dded	ed
ededededef fddZdd Z  ZS )CosineAnnealingSchedulerr   r   8 iUMu?r   r   r   r   r   constant_stepsr   r   r   r   c                    r   r   )	r!   r"   r   r   ri   r   r   r   r   )r$   r   r   ri   r   r   r   r   r%   r'   r(   r"     r)   z!CosineAnnealingScheduler.__init__c                 C   s@   ddl m} ||| j| j| j| jd}||| j| jd| jdS )Nr   )CosineAnnealing)r   r   ri   r   r,   r.   )	nemo.core.optim.lr_schedulerrj   r   r   ri   r   r   r   r   )r$   r2   r/   rj   r0   r'   r'   r(   r-     s   	z"CosineAnnealingScheduler.scheduler)r   r   rg   rh   r   r   r   )	r6   r7   r8   r:   r;   r<   r"   r-   r=   r'   r'   r%   r(   rf     s0    rf   N)typingr   rk   r   r   r   r   r   r   r	   r
   r   r   r   !nemo.lightning.pytorch.optim.baser   r   r?   rC   rN   rP   rR   rU   rV   rW   rY   re   rf   r'   r'   r'   r(   <module>   s   4)/,#)%%