o
    Ti                  
   @   s   zd dl mZ d dlmZ W n ey   d dlmZ Y nw z
d dlZd dlZW n ey: Z zW Y dZ[ndZ[ww G dd dZ	G dd deZ
dS )    )__deepspeed__)	OpBuilderNc                   @   s    e Zd ZdZdZedd ZdS )HPUFusedAdamNc                 G   s|  t jd u rddlm} ddlm} |t _| t _t j}|j  |}|	r;dt	|| }dt	|| }|t
| | }| }tj|gtjdd|d d jj|d d jdd	}|r^|
nd}
d|
|  }t jrtjj|d |d |d
 |d |||||	 n6tj|gtjdd|d d jj|d d jdd	}tjj|d |d |d
 |d ||||||dk
 |j  d S )Nr   )core)is_lazyg      ?F)dtyperequires_grad   T)non_blocking      )r   htcorehabana_frameworks.torchr   &habana_frameworks.torch.utils.internalr   is_lazy_modestep_closure_mark_step_if_lazypowmathsqrttorchtensorfloattor   deviceopshpuoptimizer_adamw)
chunk_sizenoop_flag_buffertensor_listslrbeta1beta2epsilonstepadam_w_modebias_correctionweight_decayargsr   r   	step_sizebias_correction1bias_correction2neg_step
neg_step_tmodified_wdmodified_wd_t r1   [/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/ops/op_builder/hpu/fused_adam.pymulti_tensor_adam   sf   

zHPUFusedAdam.multi_tensor_adam)__name__
__module____qualname__r   r   staticmethodr3   r1   r1   r1   r2   r      s
    r   c                       sF   e Zd ZdZdZ fddZdd Zdd Zd	d
 ZdddZ	  Z
S )FusedAdamBuilderDS_BUILD_FUSED_ADAM
fused_adamc                    s   t  j| jd d S )N)name)super__init__NAMEself	__class__r1   r2   r=   \   s   zFusedAdamBuilder.__init__c                 C   s   d| j  dS )Nzdeepspeed.ops.adam._op)r>   r?   r1   r1   r2   absolute_name_   s   zFusedAdamBuilder.absolute_namec                 C      g S Nr1   r?   r1   r1   r2   sourcesb      zFusedAdamBuilder.sourcesc                 C   rE   rF   r1   r?   r1   r1   r2   include_pathse   rH   zFusedAdamBuilder.include_pathsTc                 C   s   t S rF   )r   )r@   verboser1   r1   r2   loadh   rH   zFusedAdamBuilder.load)T)r4   r5   r6   	BUILD_VARr>   r=   rD   rG   rI   rK   __classcell__r1   r1   rA   r2   r8   X   s    r8   )
op_builderr   op_builder.builderr   ImportError deepspeed.ops.op_builder.builderr   r   er   r8   r1   r1   r1   r2   <module>   s   B