o
    ß¥µiù  ã                   @   s4   d Z ddlZddlZddlmZ G dd„ deƒZdS )z PyTorch DataLoader for TFRecordsé    N)Ú_LRSchedulerc                   @   sH   e Zd ZdZg d¢Z		ddd„Zdd„ Zdd	d
„Zdd„ Zdd„ Z	dS )ÚAnnealingLRzBAnneals the learning rate from start to zero along a cosine curve.)ÚlinearÚcosineÚexponentialÚconstantÚNoneNéÿÿÿÿc                 C   sf   || _ || _|| _|d | _|| _t|tƒr| ¡ nd | _|  	| j¡ t
j ¡ dkr1td|ƒ d S d S )Né   r   zlearning rate decaying)Ú	optimizerÚstart_lrÚwarmup_iterÚ_step_countÚend_iterÚ
isinstanceÚstrÚlowerÚdecay_styleÚstepÚtorchÚdistributedÚget_rankÚprint)Úselfr   r   r   Ú	num_itersr   Ú	last_iter© r   úZ/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/plug/AnnealingLR.pyÚ__init__   s   
ÿÿÿzAnnealingLR.__init__c                 C   s²   | j dkr| j| j krt| jƒ| j | j  S | j| jd kr-| j| j| j| j   | j  S | j| jd krK| jd t tj	| j| j   | j ¡d  S | j| jd krV| jS | jS )Nr   r
   g       @é   )
r   r   Úfloatr   r   ÚDECAY_STYLESr   ÚmathÚcosÚpi)r   r   r   r   Úget_lr-   s(   
ÿþÿÿÿzAnnealingLR.get_lrc                 C   s:   |d u r	| j d }|| _ |  ¡ }| jjD ]}||d< qd S )Nr
   Úlr)r   r%   r   Úparam_groups)r   Ústep_numÚnew_lrÚgroupr   r   r   r   @   s   

ÿzAnnealingLR.stepc                 C   s   | j | j| j| j| jdœ}|S )N©r   r   r   r   r   r+   ©r   Úsdr   r   r   Ú
state_dictH   s   ûzAnnealingLR.state_dictc                 C   sB   |d | _ |d | _|d | _|d | _|d | _|  | j¡ d S )Nr   r   r   r   r   )r   r   r   r   r   r   r,   r   r   r   Úload_state_dictR   s   




zAnnealingLR.load_state_dict)Nr	   )N)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r!   r   r%   r   r.   r/   r   r   r   r   r      s    
ú

r   )r3   r"   r   Útorch.optim.lr_schedulerr   r   r   r   r   r   Ú<module>   s
   