o Ti@sXddlZddlZddlZddlmZddlmZmZddlTddZ GdddeZ dS) N)CheckpointEngine)loggerlog_dist)*cCstjtj|SN)ospathbasenamedirname)rr p/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/checkpoint_engine/nebula_checkpoint_engine.py_get_tag_from_pathsr csLeZdZd fdd ZddZdefddZd defd d Zd d ZZ S)NebulaCheckpointEngineNcsdt|d|_d|_|j|_|j|_|jdur|j|_t|jt |j t |j i}t jdi|dS)Nr )super__init__ checkpointtag_flagenable_nebula_load load_pathnebula_load_pathpersistent_storage_pathNEBULA_PERSISTENT_STORAGE_PATHNEBULA_PERSISTENT_TIME_INTERVALpersistent_time_interval"NEBULA_NUM_OF_VERSION_IN_RETENTIONnum_of_version_in_retention torch_nebulainit)self config_paramsnebula_config_params __class__r r rs  zNebulaCheckpointEngine.__init__cCs&td|dgdt|d|_dS)Nz"[Nebula] Start Checkpoint for tag:r)ranks)rr Checkpointr)rtagr r r create&szNebulaCheckpointEngine.creatercCsjtdtd|t|}tj|}td|d|d|j ||td|d|ddS)Nz([Nebula] Create dummy files for loading.z[Nebula] Saving under tag ...z[Nebula] Saved .) rtorchsaver rrr rinfor)r state_dictrr&partition_namer r r r-,s  zNebulaCheckpointEngine.savec Cs~t|}|jdup |j|k}|js2|r2||_td|dtj||dd}td|d|Stj |}td|d|d |j d d}|d vrVt j |j d }nt j ||j d }|dusk|dur|jdkrtd|d|j dt j |j d }|dus|dur|jdkrtdt }td|ddS|j}d|_|j||d}td|d|d|j d|S)Nz6[Nebula] Disable nebula load. Loading checkpoint from z ...F) map_location weights_onlyz5[Nebula] Disable nebula load. Loaded checkpoint from z .z[Nebula] Loading r)z from nebula path r*)Nlatestlatest_universal) persist_path)r&r5r(z$Unable to find valid checkpoint tag:z= from Nebula, try to get latest checkpoint again from nebula z path!znUnable to find latest checkpoint from Nebula tier3, try to get latest checkpoint again from nebula tier1 path!z6Unable to find valid checkpoint from Nebula under tag:r+)r1z[Nebula] Loaded z from )r rrrr.r,loadrrr rrget_latest_checkpointget_checkpointr&warning)rrr1r&first_load_flag partitionr0rr r r r77s>    zNebulaCheckpointEngine.loadcCs2td|d|j}|stddS|S)Nz[Nebula] all files for z4 are saved in tier1. It is ready to start persistingz?[Nebula] failed to commit the checkpoint, please check the log.F)rr.rcommiterror)rr& commit_rlsr r r r=ds   zNebulaCheckpointEngine.commitr) __name__ __module__ __qualname__rr'strr-r7r= __classcell__r r r!r rs  -r) rr,r5deepspeed.runtime.checkpoint_engine.checkpoint_enginerdeepspeed.utilsrrdeepspeed.nebula.constantsr rr r r r s