o
    mi^                     @   s@   d dl mZ d dlmZmZ dd Z							
		dddZdS )    )Lion)AdamWAdamc                 C   s8   g g }}| D ]}|j dk r|n|}|| q||fS )N   )ndimappend)params	wd_paramsno_wd_paramsparam
param_list r   A/home/ubuntu/SpeechTokenizer/speechtokenizer/trainer/optimizer.py separate_weight_decayable_params   s
   
r   -C6?{Gz?g?gGz?:0yE>FTc                 K   s~   |dk}	|rt tdd | } |r"|	r"t| \}
}d|
i|ddg} |r,t| |||dS |	s6t| |||dS t| ||||dS )	Nr   c                 S   s   | j S )N)requires_grad)tr   r   r   <lambda>   s    zget_optimizer.<locals>.<lambda>r   )r   weight_decay)lrbetasr   )r   r   eps)r   r   r   r   )listfilterr   r   r   r   )r   r   wdr   r   filter_by_requires_gradgroup_wd_paramsuse_lionkwargshas_wdr	   r
   r   r   r   get_optimizer   s   r#   N)r   r   r   r   FTF)lion_pytorchr   torch.optimr   r   r   r#   r   r   r   r   <module>   s    	