o
    vi	                     @  sL   d dl mZ d dlmZmZ d dlZd dlmZ dd ZG dd deZ	dS )	    )annotations)TupleCallableN)	Optimizerc                 C  s   | d uS N )valr   r   H/home/ubuntu/.local/lib/python3.10/site-packages/lion_pytorch/foreach.pyexists	   s   r
   c                      s>   e Zd Z				dd fddZe 	ddddZ  ZS )Lion-C6?g?gGz?        FlrfloatbetasTuple[float, float]weight_decaydecoupled_weight_decayboolc                   sh   |dksJ t dd |D sJ t dd dD sJ d|| _|| _t|||d}t || d S )Nr   c                 S  s$   g | ]}d |  kodkn  qS )r         ?r   ).0betar   r   r	   
<listcomp>   s   $ z!Lion.__init__.<locals>.<listcomp>c                 S  s   g | ]}t td | dqS )	_foreach__)hasattrtorch)r   attrr   r   r	   r      s    )muladdsignlerpzFthis version of torch does not have the prerequisite foreach functions)r   r   r   )all_init_lrdecoupled_wddictsuper__init__)selfparamsr   r   r   r   defaults	__class__r   r	   r(      s   zLion.__init__NclosureCallable | Nonec                 C  sh  d }t |rt  | }W d    n1 sw   Y  | jD ]}|d |d g|d | j| jR \}}}}}}	|rA||	 }g }
g }g }tdd |d D ]+}|j| j| }}t	|dkrht
||d< |d }|
| || || qP|d	krt|
d
||   dd |D }t||d
|  t| tj|
|| d t||d
|  q |S )Nr   r   r   c                 S  s
   t | jS r   )r
   grad)pr   r   r	   <lambda>@   s   
 zLion.step.<locals>.<lambda>r*   r   exp_avgr   r   c                 S  s   g | ]}|  qS r   )clone)r   tr   r   r	   r   V   s    zLion.step.<locals>.<listcomp>)alpha)r
   r   enable_gradparam_groupsr%   r$   filterr0   statelen
zeros_likeappend_foreach_mul__foreach_lerp__foreach_sign__foreach_add_)r)   r.   lossgroupr   wdbeta1beta2r%   init_lrr*   gradsexp_avgsr1   r0   r:   r3   updatesr   r   r	   step&   s8   

2


z	Lion.step)r   r   r   F)r   r   r   r   r   r   r   r   r   )r.   r/   )__name__
__module____qualname__r(   r   no_gradrK   __classcell__r   r   r,   r	   r      s    r   )

__future__r   typingr   r   r   torch.optim.optimizerr   r
   r   r   r   r   r	   <module>   s    