o
    TÃi  ã                   @   s  d dl mZmZ d dlmZ d dlmZ d dlT d dlm	Z	 d dl
m
Z
mZmZmZmZmZ d dl
mZmZmZ d dl
mZmZ d d	l
mZmZmZ d d
l
mZmZmZ d dl
mZmZmZ d dl
mZ d dlm Z m!Z!m"Z"mZm#Z#m$Z$ d dl%m&Z&m'Z' ddl(m)Z) d dl*m+Z+ dS )é   )ÚloggerÚlog_dist)Úget_caller_func)ÚOnDevice)Ú*)Úinstrument_w_nvtx)Útensor_fragmentÚget_full_hp_paramÚget_hp_fragment_mappingÚfragment_addressÚget_full_hp_gradÚmap_to_flat_opt_states)Úsafe_get_full_fp32_paramÚsafe_get_full_gradÚsafe_get_full_optimizer_state)Úset_full_hp_paramÚset_full_hp_grad)Úsafe_set_full_fp32_paramÚsafe_set_full_optimizer_stateÚsafe_set_full_grad)Úsafe_get_local_fp32_paramÚsafe_get_local_gradÚsafe_get_local_optimizer_state)Úsafe_set_local_fp32_paramÚsafe_set_local_gradÚsafe_set_local_optimizer_state)Ú safe_update_full_grad_vectorized)Úset_z3_leaf_modulesÚunset_z3_leaf_modulesÚget_z3_leaf_modulesÚz3_leaf_moduleÚz3_leaf_parameterÚset_z3_leaf_module)Úlink_hp_paramsÚ#lazy_init_hp_params_optimizer_stateé    )ÚRepeatingLoader)Úget_numactl_cmdN),Úloggingr   r   Úcomms_loggingr   Úinit_on_devicer   ÚgroupsÚnvtxr   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r   r   r   r!   r"   Úmixed_precision_linkager#   r$   Údeepspeed.runtime.dataloaderr&   Únumar'   © r0   r0   úL/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/utils/__init__.pyÚ<module>   s      