o Ti@sVddlZddlZddlZddlZddlZddlZddlZddlZddlZ ddl m Z ddl m Z ddl mZddlmZddlmZmZmZmZmZmZmZmZmZmZeGdddZdZed Zd d Zd d Z ddZ!ddZ"ddZ#ddZ$ddZ%ddZ&ddZ'ddZ(ddZ)d d!Z*d"d#Z+d$d%Z,d&d'Z-Gd(d)d)Z.d*d+Z/d,d-Z0dOd/d0Z1  . .dPd1d2Z2 3 .  .dQd4d5Z3dRd6d7Z4e5d8kr)e6Z7e7j8d9e9d:d;e7j8de9d3d?d@e7j8dAd.dBdCdDe7j8dEdFe9ddGd@e7j8dHdBdIdJe7j8dKdLdBdMdJe7:Z;e;jZe3e;je;j?e;j@e;jAdNdSdS)SN)tqdm) OrderedDict) dataclass)logger) DS_VERSIONOPTIMIZER_STATE_DICTSINGLE_PARTITION_OF_FP32_GROUPSFP32_FLAT_GROUPS ZERO_STAGEPARTITION_COUNT PARAM_SHAPES BUFFER_NAMESFROZEN_PARAM_SHAPESFROZEN_PARAM_FRAGMENTSc@sFeZdZUeed<eed<eed<eed<eed<eed<dS)zero_model_statebuffers param_shapes shared_params ds_versionfrozen_param_shapesfrozen_param_fragmentsN)__name__ __module__ __qualname__dict__annotations__listintrrP/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/utils/zero_to_fp32.pyr's   rcpucCs|rt|S|SN)isdigitrtextrrratoi7sr%cCsddtd|DS)z alist.sort(key=natural_keys) sorts in human order http://nedbatchelder.com/blog/200712/human_sorting.html (See Toothy's implementation in the comments) cSg|]}t|qSr)r%).0crrr Az natural_keys..z(\d+))resplitr#rrr natural_keys;sr-cCsjtj|std|d|dkrtj|d}n |dkr%tj|d}tj|s3td|d|S) N Directory '' doesn't existzmp_rank_00_model_states.ptz)zero_pp_rank_0_mp_rank_00_model_states.ptz!can't find model states file at '')ospathisdirFileNotFoundErrorjoinexists)checkpoint_dir zero_stagefilerrrget_model_state_fileDs  r<cCsBtttj||td}t|dkrtd|d|d|S)N)keyrz can't find z files in directory 'r2)sortedglobr3r4r7r-lenr6)r9 glob_pattern ckpt_filesrrrget_checkpoint_filesTs rCcC t|dS)Nz*_optim_states.ptrCr9rrrget_optim_files^ rGcCrD)Nz*_model_states.ptrErFrrrget_model_state_filesbrHrIc sg}|D]}tj|tdd}t|vrt|d|ttr$tdfdd|dD}|t}g}|D]}| D]}| |q?q9| t d} | durctr[td| |t | 7}d d |d D} | td} | td} t||| | | | d } | | q|S) NF) map_location weights_onlyz is not a model state checkpointzFound buffers:cs"i|] \}}|vr||qSr)floatr'kv buffer_namesrr r"z&parse_model_states..modulezFound frozen_param_shapes: cSsg|]\}}||gqSrrrMrrrr)sz&parse_model_states..r)rrrrrr)torchloaddevicer ValueErrordebugprintitemsr keysappendgetrrrrr)fileszero_model_statesr; state_dictrr param_namessnamerrrr z_model_staterrPrparse_model_statesfs@       rfcst|}gt|ddD]}tj|tddd}|ddd|q tdtvr6t |dddtt}dtt }t |t urPt |}||krbt d |d |d |d |d kritn|dkrptnt d|fddttD}|||fS)NzLoading checkpoint shardsdescTF)rJmmaprKoptimizer_state_dictrz is not a zero checkpointz Expected z of '*_optim_states.pt' under 'z ' but found zu files. Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes.r0r1zunknown zero stage csg|] }|tqSr)r)r'ifp32_groups_key state_dictsrrr)sz&parse_optim_states..)r@rrUrVrWpopr]r rrXr typermaxrr range)r_ds_checkpoint_dir total_filesfrar: world_sizefp32_flat_groupsrrlrparse_optim_statess.   rxcCstd|dt|}t||\}}}td|d|t|}t|}td|dj|dkr;t||||S|dkrFt||||Sd S) z Returns fp32 state_dict reconstructed from ds checkpoint Args: - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) zProcessing zero checkpoint 'r2z'Detected checkpoint of type zero stage z, world_size: z)Parsing checkpoint created by deepspeed==rr0r1N)rZrGrxrIrfr*_get_fp32_state_dict_from_zero2_checkpoint*_get_fp32_state_dict_from_zero3_checkpoint)rsexclude_frozen_parameters optim_filesr:rvrw model_filesr`rrr)_get_fp32_state_dict_from_zero_checkpoints r~c Cs6|djdust|djdkrdS|dj}|dj}tr`tdd|D}tdtd|t|}tdd|D}tdd|D}td |d td |d |d d}d} |D]%\} } |d7}| } | | 7} || || <trt| d| d| dqhtd|d| ddS)Nrcs|]}|VqdSr!numelr'rcrrr z-_zero2_merge_frozen_params..zrank 0: .numel = csrr!rrrrrrrcSg|]}|qSrrr'prrrr)r*z._zero2_merge_frozen_params..Frozen params: Have  numels to process.Frozen params: Need numels in  params full shape:  unpartitioned numel  *Reconstructed Frozen fp32 state dict with  params elements) rr@rrYsumvaluesrZrr[r) rar`rrnum_elem wanted_params wanted_numel avail_numel total_params total_numelrdshapeunpartitioned_numelrrr_zero2_merge_frozen_paramss.    rcCst||d}t|Sr!)getattrcallable)objfnattrrrr _has_callables rc s2|dj}tr-t|D]!tt|dD]}ttdd|d||jqq t|d}g}t|D]fdd|D}t|d} | | q9t dd|D} trt dd|D} t d d|D} td | d td | d | dd} d}t ||D]\}} d}| } | D]9\}}t|dr| nt|}| |7} |d7}trt|d|d|d| d|||||<||7}qd|fdd}trtd|d| ||}|| } trtd|d| || kr td|d| dqtd|d| ddS) Nr[z][].shape=csg|]}|qSrr)r'sd)rkrrr) r*z1_zero2_merge_trainable_params..cSrrr)r'full_single_fp32_vectorrrrr)r*cSr&r)r@r'shapesrrrr)r*cSs"g|] }tdd|DqS)csrr!rr'rrrrrrz;_zero2_merge_trainable_params...)rrrrrrr)rSzHave rzNeed r params.rrrrrr0cst|Sr!mathceil)x)align_torr zero2_align2sz2_zero2_merge_trainable_params..zero2_alignzoriginal offset=z, avail_numel=zaligned offset= consumed  numels out of  - something is wrongz#Reconstructed fp32 state dict with rr)rrYrrr@rZr rrUcatr]rziprr[rrprodnarrowviewrX)rarvrwr`rjnum_param_groups&merged_single_partition_of_fp32_groupsmerged_partitionsrrrrrrroffsetrdrrrr)rrkr_zero2_merge_trainable_paramssZ  *        rcCst}|dj}||trtdt|d|s t||t|||||djD]}|d|vr>||d||d<q,|SNrzadded z buffersr) rrupdaterYrZr@rrrrvrwr`r{rarpairrrrryEs    rycCs.||}|r ||nd}t||}||fSNrr)rrv remainder padding_numelpartitioned_numelrrrzero3_partitioned_param_info\src s|djdust|djdkrdStrmt|D]}tdd||jD}td|dtd|q|dj}t|}tdd|D}tdd |djD|}td |d td |d |dd} d} |dj D]G\} | d7} | } | | 7} t fdd|D} t | ddd| | |<t| |\}}trtd| dd| d|d| qxtd| d| ddS)Nrcsrr!rrrrrrirz-_zero3_merge_frozen_params..zrank z: rcsrr!rrrrrrnrcSrrrrrrrr)or*z._zero3_merge_frozen_params..rrrrrrc3s|]}|jVqdSr!)r)r' model_staterdrrrzszFrozen params: rr partition0 numel= partitioned_padding_numel=rrr)rr@rYrrrrrrZrr[rtuplerUrrrr)rarvr`rkrrrrrrrrr param_fragsrpartitioned_padding_numelrrr_zero3_merge_frozen_paramscs6      rc@s eZdZdZddZddZdS)GatheredTensorz| A pseudo tensor that collects partitioned weights. It is more memory efficient when there are multiple groups. cCs4||_||_||_||_||_|jddj|_dSr) flat_groupsflat_groups_offsetrrrdtype)selfrrrrrrrr__init__s zGatheredTensor.__init__cCs8|j|j}t|j}g}t|D]r}|j|}d}d}tt|jD]1}|j||jkr8|j|dkr.cSs$i|]}|D]\}}||qqSr)r[)r'drNrOrrrrRs$z1_zero3_merge_trainable_params..rrcsrr!rrrrrrrz0_zero3_merge_trainable_params..zTrainable params: Have rzTrainable params: Need rrcSrrr)r'rrrrr)r*zGathering sharded weightsrgrzTrainable params: rrrrrrrz-Reconstructed Trainable fp32 state dict with rr)rrrYrrrZr rr@rrrnpcumsumrr[rrrX)rarvrwr`rrrkrrrrrrrdrrrrtensorrrr_zero3_merge_trainable_paramss>   "  rcCst}|dj}||trtdt|d|s!t|||t|||||djD]}|d|vr?||d||d<q-|Sr) rrrrYrZr@rrrrrrrrzs    rzFcCsti}i}|D]/\}}t|}||vr|||}|||<q|||<|r1tj|j|jd||<q|||<q|S)z> Convert state_dict of GatheredTensor to torch tensor )r)r[idrUemptyrrr)rareturn_empty_tensortorch_state_dictconverted_tensorsrdr tensor_id shared_tensorrrrto_torch_tensors  rcCs|dur4tj|d}tj|r-t|d}|}Wdn1s'wYntd|tj||}tj|sIt d|dt ||}|rR|St |S)a Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example via a model hub. Args: - ``checkpoint_dir``: path to the desired checkpoint folder - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` - ``exclude_frozen_parameters``: exclude frozen parameters - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. Convert the pesduo tensor to torch tensor by ``.contiguous()`` Returns: - pytorch ``state_dict`` A typical usage might be :: from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint # do the training and checkpoint saving state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu model = model.cpu() # move to cpu model.load_state_dict(state_dict) # submit to model hub or save the model to share with others In this example the ``model`` will no longer be usable in the deepspeed context of the same application. i.e. you will need to re-initialize the deepspeed engine, since ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. Note: the above usage may not work if your application doesn't have sufficient free CPU memory. You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with the checkpoint. Or you can load state_dict in lazy mode :: from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu for name, lazy_tensor in state_dict.item(): tensor = lazy_tensor.contiguous() # to cpu print(name, tensor) # del tensor to release memory if it no longer in use Nlatestrz Unable to find 'latest' file at r.r/) r3r4r7isfileopenreadstriprXr5r6r~r)r9tagr{ lazy_mode latest_pathfdrsrarrr(get_fp32_state_dict_from_zero_checkpoints-    r5GBcs|rzddlm}Wn tytdw|dur0zddlm}Wn ty/tdwt|||dd|r\}}fdd|D}t|}t j||}|r|||ddidnt||t | D]}|=||=q~tq| jr| j| jd}|rd nd!}t j||}t|d"d#d$}tj|d%dd&d'}||WddS1swYdSdS)(a Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. Args: - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - ``output_dir``: directory to the pytorch fp32 state_dict output files - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` - ``exclude_frozen_parameters``: exclude frozen parameters r) save_filezIIf you want to use `safe_serialization`, please `pip install safetensors`N)"split_torch_state_dict_into_shardszIIf you want to use `max_shard_size`, please `pip install huggingface_hub`T)rzmodel.safetensorszpytorch_model.binz.binz {suffix}.binz .safetensorsz{suffix}.safetensors)r)filename_patternmax_shard_size) namedtupleStateDictSplit is_shardedfilename_to_tensorsF)r r )exist_okzSaving checkpoint shardsrgcsi|]}||qSrr)r' tensor_namerarrrRsz>convert_zero_checkpoint_to_fp32_state_dict..formatpt)metadata)r weight_mapzmodel.safetensors.index.jsonzpytorch_model.bin.index.jsonwzutf-8)encodingr0)indent sort_keys )safetensors.torchr ImportErrorrZhuggingface_hubrrreplacer collectionsrrr\r3makedirsr r[rr4r7rUsavegccollectr rtensor_to_filenamerjsondumpswrite)r9 output_dirrsafe_serializationrr{rr weights_namerempty_state_dictstate_dict_splitrrr  shard_filetensorsshard_state_dict output_pathr indexsave_index_filerucontentrr r*convert_zero_checkpoint_to_fp32_state_dictVsr          "r0cCs8tdt||}td|}|j|dd|S)ay 1. Put the provided model to cpu 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` 3. Load it into the provided model Args: - ``model``: the model object to update - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` Returns: - ``model`: modified model Make sure you have plenty of CPU memory available before you call this function. If you don't have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it conveniently placed for you in the checkpoint folder. A typical usage might be :: from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) # submit to model hub or save the model to share with others Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context of the same application. i.e. you will need to re-initialize the deepspeed engine, since ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. zExtracting fp32 weightsz#Overwriting model with fp32 weightsF)strict)rinforr load_state_dict)modelr9rrarrr$load_state_dict_from_zero_checkpoints   r5__main__r9z?path to the desired checkpoint folder, e.g., path/checkpoint-12)rphelpr$zVdirectory to the pytorch fp32 state_dict output files(e.g. path/checkpoint-12-output/)z--max_shard_sizea@The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of sizelower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`We default it to 5GB in order for models to be able to run easily on free-tier google colab instanceswithout CPU OOM issues.)rpdefaultr7z--safe_serialization store_truezbWhether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).)r8actionr7z-tz--tagzMcheckpoint tag used as a unique identifier for checkpoint. e.g., global_step1z--exclude_frozen_parameterszexclude frozen parameters)r:r7z-dz--debugz enable debug)rr%rr{)F)NFF)rFNFr!)BargparserUr?rr3r+rr!numpyrrrr dataclassesrdeepspeed.utilsrdeepspeed.checkpoint.constantsrrrr r r r r rrrrYrWr%r-r<rCrGrIrfrxr~rrrryrrrrrzrrr0r5rArgumentParserparser add_argumentstr parse_argsargsr9r$rr%rr{rrrrs    0   .( I$.5  C U '