o Ti@s2ddlZddlZddlZddlmZmZddlmZddlZddl Z ddlm Z ddl m Z ddl Z ddlZddlmZddlmZddlmZdd lmZd d lmZdd lmZddlZd dlmZmZddlmZddlm Z m!Z!ddl"m#Z#ddl$m%Z%ddlm&Z&m'Z'ddl(m)Z)ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ddl0m1Z1d dl2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8m9Z9dgZ:da;dadXdedefdd Z?dYd"d#Z@d$eAd%dfd&d'ZBd(d)ZCd*d+ZDd,d-ZEGd.d/d/eZFGd0d1d1eZGejHZIejJZKejLZMejNZOejPZQejRZSejTZUejVZWd2ed3ejXd%efd4d5ZYd6ejXd%efd7d8ZZdZd:d;Z[e&dZ\d!a]dZ^iZ_Gd?d@d@e`ZadAdBZbdCdDZcGdEdFdFZdGdGdHdHZeGdIdJdJZfGdKdLdLZgGdMdNdNZhGdOdPdPZidQeed%eefdRdSZjGdTdUdUeaZkGdVdWdWZldS)[N)CallableIterable)Enum)List) defaultdictTensor)comm)Module) Parameter)zero3_linear_wrap)groups)see_memory_usageget_only_unique_item)DeepSpeedZeroConfig)assert_ints_same_as_other_ranks is_zero_param)OffloadDeviceEnum)get_config_default)instrument_w_nvtxlogger)init_distributed)debug_param2name_id_shape debug_param2name_id_shape_devicedebug_module2namedebug_param2name_id debug_param2name_id_shape_status)get_accelerator) AsyncPartitionedParameterSwapperPartitionedParamStatus)_quantize_paramWEIGHT_QUANTIZATION_LAYERSwrap_quantized_functionalwrap_load_from_state_dictc@s(eZdZdeddfddZdddZdS) NoGatherHandleparamreturnNcCs|jtjkrtd|dt|jdr/tj |jj |jj j t dd|j|_ n|jj j t dd|j|_ ||_dS)Nexpected param  to be availableds_quant_scaleTdevice non_blocking) ds_statusZeroParamStatusINFLIGHT RuntimeError ds_summaryhasattr ds_tensorInitquantizer_module dequantizedatar+torcurrent_device_nameviewds_shape_NoGatherHandle__paramselfr'rA_/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/zero/partition_parameters.py__init__.s    zNoGatherHandle.__init__cKs&ts ttj|j_dSN)rresolves_data_dependencycurrent_stream synchronizer0 AVAILABLEr>r/)r@kwargsrArArBwait:s zNoGatherHandle.waitr(N__name__ __module__ __qualname__r rCrJrArArArBr&,s r&c@0eZdZdeeddfddZedddZdS) NoGatherCoalescedHandleparamsr(NcCs||_d|_|jD]B}|jtjkrtd|dt|jdr:t j |jj |jj jtdd|j|_ q |jj jtdd|j|_ q dS)NFr) to not be availabler+Tr,) _NoGatherCoalescedHandle__params"_NoGatherCoalescedHandle__completer/r0r1r2r3r4r5r6r7r8r9r+r:rr;r<r=)r@rRr'rArArBrCBs"    z NoGatherCoalescedHandle.__init__cKs`|jrdStst|jD]}|jtjks&Jd| dtj |_qd|_dSNr) to be inflightT) rUrrErFrGrTr/r0r1r3rH)r@rIr'rArArBrJPs     zNoGatherCoalescedHandle.waitrKrMrNrOrr rCrrJrArArArBrQ@srQ input_tensor output_tensorcCsttj|||ddS)NTgroupasync_op)rdist allgather_fn)rYrZr\rArArB_dist_allgather_fn^sr`FcCs,t}|dkr|s |rt|dSdSdSNr)r^get_rankprint)messagedebugforcerankrArArB print_rank_0bs rhmsgr(cCstdkr t|dSdSra)r^rbrre)rirArArB debug_rank0ms rjcsHt|ds"i|_ddfdd}t||_t|||_dSdS)N_external_paramscSs |jSrD)rkitemsr@rArArBexternal_parametersv z2_init_external_params..external_parameterscst|j|dd|S)NFrecurse) itertoolschainnamed_parametersrmrnrArBall_parametersysz-_init_external_params..all_parameters)r4rktypes MethodTypeds_external_parametersrv)modulervrArurB_init_external_paramsrs  r{cCs>t|tjjs tdt|dst|t|}||j|<dS)aInstruct DeepSpeed to coordinate ``parameter``'s collection and partitioning in the forward and backward passes of ``module``. This is used when a parameter is accessed outside of its owning module's ``forward()``. DeepSpeed must know to collect it from its partitioned state and when to release the memory. .. note:: This is only applicable to training with ZeRO stage 3. Args: module (``torch.nn.Module``): The module that requires ``parameter`` in its forward pass. parameter (``torch.nn.Parameter``): The parameter to register. Raises: RuntimeError: If ``parameter`` is not of type ``torch.nn.Parameter``. Examples ======== #. Register a weight that is used in another module's forward pass (line 6). Parameter ``layer1.weight`` is used by ``layer2`` (line 11). .. code-block:: python :linenos: :emphasize-lines: 6,11 class ModuleZ3(torch.nn.Module): def __init__(self, *args): super().__init__(self, *args) self.layer1 = SomeLayer() self.layer2 = OtherLayer() deepspeed.zero.register_external_parameter(self, self.layer1.weight) def forward(self, input): x = self.layer1(input) # self.layer1.weight is required by self.layer2.forward y = self.layer2(x, self.layer1.weight) return y %Parameter is not a torch.nn.ParameterrkN) isinstancetorchnnr r2r4r{idrkrz parameterkeyrArArBregister_external_parameters * rcCsJt|tjjs tdt|drt||jvrtdt|}|j|=dS)aReverses the effects of :meth:`register_external_parameter`. Args: module (``torch.nn.Module``): The module to affect. parameter (``torch.nn.Parameter``): The parameter to unregister. Raises: RuntimeError: If ``parameter`` is not of type ``torch.nn.Parameter``. RuntimeError: If ``parameter`` is not a registered external parameter of ``module``. r|rkz;Parameter is not a registered external parameter of module.N)r}r~rr r2r4rrkrrArArBunregister_external_parameters   rc@eZdZdZdZdZdS) ZeroParamTyper rN)rMrNrONORMAL PARTITIONEDREMOTErArArArBrsrc@r)r0r rrN)rMrNrOrH NOT_AVAILABLEr1rArArArBr0sr0fntarget_fp_dtypecsdtffdd }|S)Nr(csT|dddurtttjd|d<|i|}|r(|j |_|S)Nr- LOCAL_RANK) getr~r-r device_nameosenvironis_floating_pointr9r:)argsrItensorrrrArB wrapped_fns z:zero_wrapper_for_fp_tensor_constructor..wrapped_fnr)rrrrArrB&zero_wrapper_for_fp_tensor_constructors rdtypecsdtffdd }|S)Nr(csNtttjd}|sd}td|dj|i|}|r%| }|S)Nrrr)r-) r~r-rrrr_orig_torch_empty new_emptyrr:)clsrrIr-rrrArB new_tensors z/get_new_tensor_fn_for_dtype..new_tensorr)rrrArrBget_new_tensor_fn_for_dtypes rTcs4gfdd|t}|r|||S)Ncs$|D] }||qdSrD)__subclasses__append)clsubclassrq subclass_listrArBrq s   z#get_all_subclasses..recurse)setadd)r include_rootretrArrBget_all_subclasses s rr'cCs^|jr J|t|jrts|jttj d|j |j d|_t j |_dS)z'Free underlying storage of a parameter.rrr-N)ds_active_sub_modulesr3ron_acceleratorr9is_synchronized_device record_streamrFr~emptyrr-r0rr/r'rArArB free_params   rc@s^eZdZdZdZdddZddZdd Zd d Zd d Z ddZ ddZ ddZ ddZ dS)&InsertPostInitMethodToModuleSubClassesrTNcCs||_||_||||jtjtjtjfvs Jd|jdt|_ d|_ d|_ |dur<|j r>|j j r@|j j |_ dSdSdSdS)NzInvalid data type z>, allowed values are [torch.half, torch.bfloat16, torch.float]r) mem_efficient_linearenabled _set_dtyperr~halfbfloat16floatr wrapped_clsskip_init_depthquantized_initializationweight_quantization_config)r@rr ds_configrrArArBrC2s    z/InsertPostInitMethodToModuleSubClasses.__init__cCs*|jsdStdkr||atd7adS)Nrr )rzero_init_contextpatch_init_and_builtinstop_level_contextrmrArArB __enter__@s  z0InsertPostInitMethodToModuleSubClasses.__enter__cCsl|jsdStd8atdkr.|datdkr.tjd}tj}t d|d|dd|dur4dSdS) Nr rgeAz+finished initializing model - num_params = z, num_elems = z.2fBF) rrunpatch_init_and_builtinsrr^rbrnum_module_elementsnum_module_parametersrinfo)r@exc_type exc_value traceback billion_elems num_paramsrArArB__exit__Ls  z/InsertPostInitMethodToModuleSubClasses.__exit__cCsdSrDrA)r@rzrArArB_post_init_methoddsz8InsertPostInitMethodToModuleSubClasses._post_init_methodcCs|dur*|dur*|jr|jrtd|jrtj|_dS|jr$tj|_dStj|_dSt r4|p3tj n tj r;tjntj |_dS)Nz+bfloat16 and fp16 cannot be enabled at once) bfloat16_enabled fp16_enabledr2r~rrrrris_fp16_supportedfloat16is_bf16_supportedfloat32)r@rrrArArBrgs    z1InsertPostInitMethodToModuleSubClasses._set_dtypecspdtdtfdd}fddfddfd d }fd d }ttjjjjD]}||q*tjjjjjtjjjj_tjjjjj tjjjj_ tj j tj _ t|tjjjj_tjri|tjjjjj tjjjj_ jrtd ddttdstjjjt_ttjj_jrtdddttjjjtjj_ttjjjtjj_tD]}t|j|_qt !dd_"dS)Norig_module_apply_fnr(cs<dtdtfddtdtdtddffdd }|S) amany models make use of child modules like Linear or Embedding which perform their own weight initialization in their __init__ methods, but will then have more weight initialization in a parent module's __init__ method that modifies weights of child modules, which is typically done using the Module.apply method. since the Init context manager partitions child modules immediately after they are initialized, without modifying apply we would entirely skip any initialization done by parent modules. to get around this issue, we wrap the function passed to Module.apply so that the applied function is applied to child modules correctly. fn_to_applyr(cs8tdrStdtddffdd }d|_|S)Nwrappedmodule_to_apply_fn_tor(csxttdd|jddDddd}|D]}|q||D] }tj|jd|jd q"|D]}|jd d q1d S) agathers parameters before calling apply function. afterwards parameters are broadcasted to ensure consistency across all ranks then re-partitioned. takes the following steps: 1. allgathers parameters for the current module being worked on 2. calls the original function 3. broadcasts root rank's parameters to the other ranks 4. re-partitions the parameters cSsg|]}t|r|qSrAr.0prArArB zInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..apply_with_gather..get_wrapped_fn_to_apply..wrapped_fn_to_apply..FrpcS|jSrDds_idrrArArBzInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..apply_with_gather..get_wrapped_fn_to_apply..wrapped_fn_to_apply..rrr\T)has_been_updatedN) listsorted parameters all_gatherr^ broadcastr9ds_process_group partition)rparams_to_apply_fn_tor'rrArBwrapped_fn_to_applys zInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..apply_with_gather..get_wrapped_fn_to_apply..wrapped_fn_to_applyT)r4 functoolswrapsr r)rrrArrBget_wrapped_fn_to_applys !zzInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..apply_with_gather..get_wrapped_fn_to_applyrzNcs||dSrDrA)rzrrrrArB wrapped_applyszpInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..apply_with_gather..wrapped_apply)rrrr )rrrArrBapply_with_gatherxs*zYInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..apply_with_gathercs\fddfdd}fdd}ddttjjjjD]}||q|j_dS) Ncstfdd}|S)Ncs$|g|Ri|}||SrD)r)rzrrI_module)fr@rArBwrappers zInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..hook_for_skip_init..partition_after_empty_init..wrapperrrrrrmrrBpartition_after_empty_initsz~InsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..hook_for_skip_init..partition_after_empty_initcstfdd}|S)Ncs6|i|}ttjjjjD]}|q_|SrDrr~rmodulesrzr to_empty)rrIresr)_disable_class_applyrrzrArBrs  zInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..hook_for_skip_init..post_wrapper_to_empty..wrapperrr)rrzrrBpost_wrapper_to_emptys zyInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..hook_for_skip_init..post_wrapper_to_emptyc&d|jvr|j|_|j|_dSdS)N_apply)__dict__r_old_apply_of_skip_init_hookr)rrArB_enable_class_apply zwInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..hook_for_skip_init.._enable_class_applycSt|dr |j|_dSdS)Nr )r4r rr rArArBr  zxInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..hook_for_skip_init.._disable_class_applyr)rzrr rrm)rrzrrBhook_for_skip_inits   zZInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..hook_for_skip_initcstfdd}|S)Ncstd|jjddd}t|dsd}t|ddd|vo#|ddk}|r-jd7_|g|Ri||rCjdkrC||r_t|dtd |jjddjd kr_|td |jjdd|rujd8_dSdS) NzBefore initializing Frf_ds_child_enteredTr-metar zRunning post_init for rz-After initializing followed by post init for )rh __class__rMr4setattrrdelattrr)rzrrIis_child_module init_on_meta)rrr@rArBrs(      zhInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..partition_after..wrapperrr)rr@rrBpartition_afters&zWInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins..partition_aftercrNrCr rC _old_initr rrArB _enable_classr zUInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins.._enable_classcrrr)rrIrrArB_init_subclassr zVInsertPostInitMethodToModuleSubClasses.patch_init_and_builtins.._init_subclasszwnn.functional.linear has been overridden with a more memory efficient version. This will persist unless manually reset.Fr linear_bkzGnn.functional.linear has been overridden with quantized linear version.z+Enable Zero3 engine with INT4 quantization.T)#rrr~rrrzr __init_subclass___old_init_subclassapply _old_applyr__new__ __old_new__ classmethodr6override_module_apply_add_tensor_creation_wrappersrrhr4r functionallinearr r rr$ embeddingr#r%_load_from_state_dictrrpatched)r@rrrrrrA)rrr@rBrvs< ?/ +         z>InsertPostInitMethodToModuleSubClasses.patch_init_and_builtinscCsx|jr:dd}ttjjjjD]}||qtjjjjjtjjjj_t j r1tjjjjj tjjjj_ | d|_dSdS)NcSr)Nr)r4rrCr rArArB_disable_classArzXInsertPostInitMethodToModuleSubClasses.unpatch_init_and_builtins.._disable_classF)r.rr~rrrzr r"r!r6r(r$r# _remove_tensor_creation_wrappers)r@r/rrArArBr>s  z@InsertPostInitMethodToModuleSubClasses.unpatch_init_and_builtinscCst|jtj_tt|jt_tt|jt_ tt |jt_ tt |jt_ tt|jt_tt|jt_tt|jt_tt|jt_dSrD)rrr~rr%r_orig_torch_tensorrrr_orig_torch_zeroszeros_orig_torch_onesones_orig_torch_fullfull_orig_torch_arangearange_orig_torch_eyeeye_orig_torch_randnrandnrmrArArBr)QszDInsertPostInitMethodToModuleSubClasses._add_tensor_creation_wrapperscCs@tjjtj_tt_tt_tt_ t t_ t t_ tt_tt_tt_dSrD)r~rr&r%r1rrrr2r3r4r5r6r7r8r9r:r;r<r=rmrArArBr0\s  zGInsertPostInitMethodToModuleSubClasses._remove_tensor_creation_wrappers)TTNN)rMrNrOrrrCrrrrrrr)r0rArArArBr.s  I rcCtrtdSdS)z This function is used to initialize deepspeed engine inside the context of Init. We need to remove the wrappers but keep the context. N)rrrArArArBshutdown_init_contexths r?cCr>)z^ This function is used to restore the wrappers after deepspeed engine is initialized. N)rrrArArArBrestore_init_contextqs r@c@s,eZdZd deddfddZd d ddZdS) AllGatherHandleNr'r(cCs6|jtjkrtd|d||_||_||_dS)Nr)r*)r/r0r1r2r3_AllGatherHandle__handle_AllGatherHandle__param_AllGatherHandle__quantization)r@handler' quantizationrArArBrC{s  zAllGatherHandle.__init__TcCsXt|jj|jr%t|jjj|jj|jj|jj |j j |j _ t j|j _dSrD)rrBrJrD quant_handlebackendr8quantized_param scale_bufferr:rCr-r9r0rHr/)r@handle_dependencyrArArBrJs  zAllGatherHandle.waitrDTrKrLrArArArBrAysrAc @sTeZdZgZ  ddeedeededdfddZe ddd d Z e d d Z dS)AllGatherCoalescedHandleFNrR partitions world_sizer(cCsZ||_||_||_||_||_d|_||_|jD]}|jtj kr*t d| dqdSNFr)rS) allgather_handlerRrNrOuse_secondary_tensorcompleterFr/r0r1r2r3)r@rQrRrNrOrRrFr'rArArBrCs   z!AllGatherCoalescedHandle.__init__Tc Cs|jrdSt|jj|jrFt|jjj|jj|jj|jj  |j dj }g|_ t|jD]}|j |d|jj||jjq2d}|j D]n}|jtjks]Jd|dg}|jj}|jrk||j9}t|jD] }||} | |jkr|j |d|t|j| |} || qpttj||j|_ tj!|_t"#s|r|D] } | $t"%q||7}qKd|_t"#s|st&j'|dSdSdS)Nrr)rWT)(rSrrQrJrFrGrHr8rIrJr:rRr-rNrangerOrnarrow partition_szr/r0r1r3r5ds_numelrR!ds_secondary_tensor_num_of_groupsminr~catr<r=r9rHrrrrFrM data_buffer) r@rK flat_tensori param_offsetr'rNds_tensor_numelrg param_start part_to_copyrArArBrJsP       zAllGatherCoalescedHandle.waitcCs gt_dSrD)rMr[rArArArB free_buffers z$AllGatherCoalescedHandle.free_bufferFNrLrK) rMrNrOr[rr rintrCrrJ staticmethodrbrArArArBrMs" +rMc@s*eZdZdeefddZd d ddZdS) MultipleAllGatherHandleshandlescCs ||_dSrD)rg)r@rgrArArBrCroz!MultipleAllGatherHandles.__init__Tr(NcCs|jD]}||qdSrD)rgrJ)r@rKrErArArBrJrzMultipleAllGatherHandles.waitrLrK)rMrNrOrrMrCrJrArArArBrfsrfc@rP) AllReduceCoalescedHandlerRr(NcCsB||_||_d|_|jD]}|jtjkrtd|dq dSrP)rErRrSr/r0r1r2r3)r@rErRr'rArArBrCs  z!AllReduceCoalescedHandle.__init__cCsV|jrdSt|jj|jD]}|jtjks!Jd|dtj |_qd|_dSrV) rSrrErJrRr/r0r1r3rHr?rArArBrJs    zAllReduceCoalescedHandle.waitrKrXrArArArBrhs rhc@seZdZdddZdS)QuantizationInfor(NcCsd|_d|_d|_d|_dSrD)rIrHrGrJrmrArArBrCs zQuantizationInfo.__init__rK)rMrNrOrCrArArArBrisric@s:eZdZdZdZeZdZd ddZd ddZ d d Z dS) CUDAQuantizerTi@Nr(cCs$tjdurtjjt_dSdSrD)rjquantizer_cuda_module deepspeedops op_builderQuantizerBuilderloadrmrArArBrCs zCUDAQuantizer.__init__cCsT|durz |j|}Wntyt||j}||kr9|d|dkr/n |d7}||ks$ |d|ddkrT|||jkrT|d9}nnq:|d|dkslJd|d|||dksJ|d |d ||ksJd |||j|<Ynw|j|t |d|jj S) Nrr TrzGQantized weight requires the number of weights be a multiple of 8. Yet z cannot be divided by 8*i>z / z is larger than 16kzNAdaptive grouping algorithm cannot find a group size for input tensor of size ) group_size_cachenumelKeyErrormathceiltarget_group_sizerkquantizer:rr Symmetric)r@r'rrArArBrx sB    ( zCUDAQuantizer.quantizecCs|j|||d|jjS)Nrq)rkr8rsry)r@rIscalerArArBr8&szCUDAQuantizer.dequantizerKrD) rMrNrO async_flagrwdictrrrkrCrxr8rArArArBrjs   rjrRcCsb|D]}|jtjkrtd|tj|_qt|ddd}t|dkr-|\}t|St |S)Nz.rr ) r/r0rr2r3r1rlenr&rQ)rRr'rArArB_no_gather_coalesced+s   r~csdeZdZdZeedZeedZdZdZ dZ eedZ               d>fdd Z d d Z d d ZddZddZddZddZddZddZddZddZed?ddZd@d d!ZedAd"d#ZedBd$d%Zd&d'ZdCd(d)ZdDd*d+Ze dEd,d-Z!d.d/Z"d0d1Z#dBd2d3Z$dBd4d5Z%d6d7Z&d8d9Z'e(d:d;Z)d||j?|_@nd|_@|durtA|t#jBjCs{J|D|jEd dt F|_G|jGstHdt#jItJtKd|_L|dur|jjL|_LdSdS)aA context to enable massive model construction for training with ZeRO-3. Models are automatically partitioned (or, sharded) across the system and converted to half precision. Args: module (``torch.nn.Module``, optional): If provided, partition the model as if it was constructed in the context. data_parallel_group (``deepspeed.comm`` process group, optional): The group of processes to partition among. Defaults to all processes. Synonymous with sequence data parallel group for param partitioning across both sequence and data parallel groups. mem_efficient_linear (bool, optional): Replace torch.nn.functional.linear with an implementation that allows DeepSpeed to partition parameters. Defaults to ``True``. remote_device (string, optional): The initial device to store model weights e.g., ``cpu``, ``nvme``. Passing ``"cpu"`` will create the model in CPU memory. The model may still be moved to GPU based on the offload settings for training. Defaults to param offload device if a config is defined, otherwise GPU. pin_memory (bool, optional): Potentially increase performance by using pinned memory for model weights. ``remote_device`` must be ``"cpu"``. Defaults to pin_memory value in config, otherwise ``False``. config_dict_or_path (dict or ``json file``, optional): If provided, provides configuration for swapping fp16 params to NVMe. config (dict or ``json file``, optional): Deprecated, use config_dict_or_path instead. enabled (bool, optional): If ``False``, this context has no effect. Defaults to ``True``. dtype (``dtype``, optional): Can be used to change the data type of the parameters. Supported options are ``torch.half`` and ``torch.float``. Defaults to ``None`` mpu (``object``, optional): A model parallelism unit object that implements get_{model,data}_parallel_{rank,group,world_size}. zero_param_parallel_group(``object``, optional): Parallel (comm) group for dual partitioning of ZeRO params. zero_quantized_weights (bool, optional): If ``True``, turn on quantized weights in all gather weights. Default is ``False`` zero_quantized_nontrainable_weights (bool, optional): If ``True``, nontrainable weights will be stored in quantized format. Default is ``False`` param_swapper (``deepspeed.runtime.swap_tensor.partitioned_param_swapper.AsyncPartitionedParameterSwapper``, optional): [Experimental] Use existing parameter swapper. Defaults to ``None``. This argument will be removed in the near future. This context accelerates model initialization and enables models that are too large to allocate in their entirety in CPU memory. It has the following effects: #. allocates tensors to either GPU or CPU memory or NVMe #. converts floating point tensors to half precision #. immediately partitions tensors among the group of data-parallel devices #. (*optional*) replaces ``torch.nn.functional.linear`` with a more memory-efficient implementation These modifications allow for models that exceed the size of local CPU/GPU memory/NVMe, but fit within the total NVMe capacity (*i.e.*, aggregate CPU or GPU memory or NVMe) across all nodes. Consider initializing a model with one trillion parameters, whose weights occupy two terabytes (TB) in half precision. The initial CPU allocation in full precision requires 4TB of memory *per process*, and so a system with 8 GPUs per node would need 32TB of CPU memory due to data-parallel redundancies. Instead, by immediately partitioning tensors we remove the redundancies. The result is that regardless of the number of GPUs, we still only require the original 4TB. This allows for a linear increase in model size with the aggregate system memory. For example, if a node has 1TB of memory and 8 GPUs, we could fit a trillion parameter model with 4 nodes and 32 GPUs. Important: If the fp16 weights of the model can't fit onto a single GPU memory this feature must be used. .. note:: Initializes ``deepspeed.comm`` if it has not already been done so. See :meth:`deepspeed.init_distributed` for more information. .. note:: Only applicable to training with ZeRO-3. Examples -------- #. Allocate a model and partition it among all processes: .. code-block:: python with deepspeed.zero.Init(): model = MyLargeModel() #. Allocate a model in pinned CPU memory and partition it among a subgroup of processes: .. code-block:: python with deepspeed.zero.Init(data_parallel_group=mpu.get_data_parallel_group(), remote_device="cpu", pin_memory=True): model = MyLargeModel() #. Partition an already-allocated model in CPU memory: .. code-block:: python model = deepspeed.zero.Init(module=model) NzYzero.Init: the `config` argument is deprecated. Please use `config_dict_or_path` instead.)rrrrzBParameters cannot be scattered without initializing deepspeed.commzcsequence_data_parallel_group' is deprecated and will be removed. Use 'data_parallel_group' instead.zyBoth 'data_parallel_group' and 'sequence_data_parallel_group' were specified. Please provide only one of these arguments.rr zhpZeRO group size: TrzihpZeRO partition parameter my rank in world {} my rank in group {} ranks in my param partition group: {} rzUsing quantizer for weights: Frpz5all_gather_into_tensor API is not available in torch use_all_reduce_for_fetch_params)MrwarningrlruntimeconfigDeepSpeedConfig zero_configmemory_efficient_linearsuperrCr^is_initializedrget_world_groupr ValueErrorrbrgget_world_size dp_world_sizezero_param_process_groupzero_hpz_partition_sizer!_create_zero_param_parallel_group$_get_zero_param_intra_parallel_groupnum_ranks_in_param_group rank_in_groupnum_param_groups/_get_zero_param_intra_parallel_group_world_sizerd._get_zero_param_intra_parallel_rank_in_mygrouprhreformat*_get_zero_param_intra_parallel_group_ranksr~r-rrrr local_device set_devicequantized_weightszero_quantized_weightsquantized_nontrainable_weights#zero_quantized_nontrainable_weightsrzrjr7rrMr(r6 offload_param pin_memory_validate_remote_devicernone remote_devicecpunvmer r param_swapperr}rr _convert_to_zero_parametersrhas_all_gather_into_tensoruse_all_gather_into_tensorr __version__rrr)r@rzdata_parallel_grouprrrconfig_dict_or_pathrrrmpuzero_param_parallel_grouprrsequence_data_parallel_groupr _ds_configrrArBrCBsp                 z Init.__init__cCs$dt_|jjt_|jj|jt_dS)NT)r6apply_param_persistencerrrnum_partitions)r@rrArArB_update_persist_configs zInit._update_persist_configcCs\||t|krt|jd|nt|jt|d||dSra)_convert_to_deepspeed_paramr^rget_dp_process_grouprr9get_global_rankrr?rArArB_zero_init_params  zInit._zero_init_paramcCs2|D]}t|r q|j|j|_||qdSrD)rr9r:rr)r@ param_listr'rArArBr%s  z Init._convert_to_zero_parameterscCs|durI|dtjfvr&|jjdur&|jjj}|tjks&Jd|d|d|tjkrK|jjdus:Jdtjd|jjjdusMJdtjdSdSdS)Nz''device' in DeepSpeed Config cannot be z if remote device is .zH"offload_param" must be defined in DeepSpeed Config if remote device is zC"nvme_path" in DeepSpeed Config cannot be None if remote device is )rrrrr-r nvme_path)r@rroffload_param_devicerArArBr-s"      zInit._validate_remote_devicecCstd|jjddtd|jjdd|jddD]Z\}}td|d|jjddtjd7_tj|7_t |svt |sP|j |j|_ |d krc|jrct|tvrct||j||td t|d t|qtd tjd |jjdddS)NzConverting Params in Frz-Before converting and partitioning params in rpzAnalyzing param z in r weightPartitioning param z module=z Param count z.. After converting and partitioning params in )rhrrMrrtrrrrsrrrr9r:rrtyper#r"rrr)r@rznamer'rArArBr<s(    zInit._post_init_methodc stj_tj_j__ d_ t _ t jr=j t jkr=t jj t jkr=d_t jd7_t jj 7_nd_d_j_d_j_j_j_j_t j_ t jd7_d&fdd }ddt!  d'd t"t#d t$d t$d t%ffd d }d(fdd }d)fdd }d*fdd }fdd}fdd}fdd} fdd} d+dt&j'dt$d t(fdd } fd!d"} d#t)d t)ffd$d% } |_*|_+|_,|_-|_.|_/|_0| _1t23| _4| j5_5| _6dS),NTr Frcs"}|dur |g}j|||dS)Nr] hierarchy) _all_gather)rr]rrr'r@rArBrsz4Init._convert_to_deepspeed_param..all_gatherc Sstdd|D}|djdu}|rtdd|D}tj|||tdd}g}t|D]} ||d|| |q/|rQt tj dd|D||d nt tj d d|D||d t ||||} t | ||||d S) Ncs|]}|jjVqdSrDr5rWrrArArB zNInit._convert_to_deepspeed_param.._all_gather_dtype..rcs|] }|jj|jVqdSrDr5rWrXrrArArBrsFrr- requires_gradcSg|] }|jtqSrAds_secondary_tensorr:rr;rrArArBrzOInit._convert_to_deepspeed_param.._all_gather_dtype..)outcSrrAr5r:rr;rrArArBrr)rQrRrNrOrR) sumrr~rrr;rTrrUrrZr`rM) rrRrOrrrVrRr\rNr]rErArArB_all_gather_dtypes<   z;Init._convert_to_deepspeed_param.._all_gather_dtyperR safe_moderxr(c s"|jdkrt|S|D]}|jtjkrt|tj|_qj }j }j }|dj du}j rA|rAj }j}j}t|ddd}ttjr[tddd|D|rotd d|Dtd d|Dt|dkr.|d}t|j||}|r|j jd|}|r|j n|j} tj||s| jntj t!"d d } |st#| $t!"| |} | %dd|j&|j'$|j(|_)t*| |St+| d r| j,} | j)} nj-.| \} } t#| $t!"| |}tj| /|| jt!"d d }t#| $t!"||}t0}| %dd|j&|j'$|j(|_1j-|_2||_3||_4t*|||dSj5r|s|st6dd|D}tj7|t8dd|Dt!"d d }d}|D],}|%d||j&|j'|_)||jj9}|%d||jj:|j||j7}qXt;j<||dd}t=||dS|st>t?}|D] }||jj@|qg} |AD]\}}| @|||||qtB| St6dd|D}|rt6dd|D}tj||tj t!"d d }|r t+|dj d r tCtjDdd|D} tCtjDdd|D} nGj-.tCtjDdd|D\} } n4t+|djd rBtCtjDdd|D} tCtjDdd|D} nj-.tCtjDdd|D\} } tj| /|tjEt!"d d }t#| ||}t#| ||}t0}||_1j-|_2||_3||_4||_F||_GtH||d|||dS)Nr rcSrrDrrrArArBrrzPInit._convert_to_deepspeed_param..all_gather_coalesced..rz-allgather_coalesced: cSg|]}|jqSrArrrArArBrzRInit._convert_to_deepspeed_param..all_gather_coalesced..cSrrArrrArArBrrcSg|]}|jjqSrArrrArArBrFrr+)rFcss|]}|jVqdSrD)ds_numel_alignedrrArArBrszQInit._convert_to_deepspeed_param..all_gather_coalesced..csrrD)r5rrrArArBrrTr[)rErRcsrrDrrrArArBr-rcsrrDrrrArArBr0scS g|] }|jjtqSrA)rr9r:rr;rrArArBr:cSrrA)rr+r:rr;rrArArBr>rcSrrArrrArArBrDscSrrA)r5r9r:rr;rrArArBrKs cSrrA)r5r+r:rr;rrArArBrLrcSrrArrrArArBrSr)rQrRrNrOrRrF)I*_ensure_availability_of_partitioned_paramsrr~r/r0rr2r3r1rrgrrrrrrr isEnabledForloggingDEBUGrjrr}rurvrWshaper5r~rrint8rr;r`r:rUr<r=r-r9rAr4r+r7rxrsrirIrHrGrJrrr3rget_partition_rankcopy_r^ all_reducerhrrrrlrfrrZrrVrOrM)rRrrxr'rrrOrR buffer_sizeparam_ds_tensor param_bufferrgscalesrIrEquant_scale_bufferrG quant_infoflat_buffer_sizer\ start_paramstart dtype_paramsrrrV)rr@rArBall_gather_coalesceds(       "             z>Init._convert_to_deepspeed_param..all_gather_coalescedcsD}td|dt|dd|dur|g}j||dddS)N--z----Partitioning param FrTr free_data)rhr _partition)rrrrrrrArBrlsz3Init._convert_to_deepspeed_param..partitioncsB}|dur |g}td|ddd|Dd|dS)Nrz*----Reducing Gradients for param with ids cSrrArrr'rArArBryrzWInit._convert_to_deepspeed_param..reduce_gradients_at_owner..z to owner)rh_reduce_scatter_gradients)rrrrrArBreduce_gradients_at_ownertszCInit._convert_to_deepspeed_param..reduce_gradients_at_ownercsR}td|dt||dur|g}t|tjr|g}j|||ddS)Nrz(----Partitioning param gradient with id )partition_buffers accumulate)rhrr}r~r_partition_gradients)rrrrrrrArBpartition_gradients}s z=Init._convert_to_deepspeed_param..partition_gradientsc SrD) _aligned_sizerArrArB aligned_sizeroz6Init._convert_to_deepspeed_param..aligned_sizecrrD) _padding_sizerArrArB padding_sizeroz6Init._convert_to_deepspeed_param..padding_sizecrrD)_partition_numelrArrArBpartition_numelroz9Init._convert_to_deepspeed_param..partition_numelcsSrD)r _orig_itemrArrArB item_overridesz7Init._convert_to_deepspeed_param..item_overrideslfuse_debug_namec Ssr|rt|n|j|jj||jt|jt|j|j |j dur%t|j jnd|j |j |j dur5|j jd Sdd S)N) rstatusrsrWrr=r grad_shapepersistactive_sub_moduleszds_tensor.shape)rrr/rrsrWtuplerr=rgrad ds_persistrr5)rrrArArBr3s z4Init._convert_to_deepspeed_param..ds_summarycs|dSrD)rrrmrArBconvert_to_zero_parametersszDInit._convert_to_deepspeed_param..convert_to_zero_parametersfunccsfdd}|S)Ncs|i|SrD)r)rrI)rr'rArBrszKInit._convert_to_deepspeed_param..allgather_before..wrappedrA)rrr)rrBallgather_beforesz:Init._convert_to_deepspeed_param..allgather_before)NFrFF)NrFTra)NNrF)F)7rr ds_param_typer0rHr/rr=rsrWr5rrr6rrnum_persisted_elementsrr num_persisted_parametersis_external_paramrrrds_zero_param_process_grouprds_secondary_tensor_group_sizerrXr nvme_swapperparam_idrrrr boolrMr~rr|rrrrrrrrrrwrxr3itemr)r@r'rrrrrrrrrr3rrrA)rr'r@rBrTsl $# =       z Init._convert_to_deepspeed_paramcCs|j||SrD)rWrr?rArArBrszInit._aligned_sizecCs|j|j}|r |j|SdSra)rWr)r@r' remainderrArArBrs zInit._padding_sizecCs|jjSrDrr?rArArBrszInit._partition_numelcCsg}g}|D]8}|jjtjkr#|jjtjkr|jtjksJ| ||jjtj kr>|jjtjkr7|jtjks9J| |qt |dkrQ|dj j |dddSt |dkr`|dj dSdS)NrF)r])r5rr!rfinal_locationrrr/r0rr1r}rswap_insynchronize_reads)r@rR swap_in_listswap_in_flightr'rArArBrs    z/Init._ensure_availability_of_partitioned_paramsc Cs||g}g}|D]!}|jtjkr,|r'|j|||d}tj|_||q ||q |s}t|dkr=|j||d}n5g} g} |D]}t |j dsVt |dr\t |j dr\| |qC| |qC|j | |dd|j | |dd|D]}tj |_qtdS|S) Nrr )rr+rF)rxT)rr/r0r_allgather_paramr1rr}_allgather_paramsr4r5r_allgather_params_coalescedrH) r@rr]rrgall_gather_listr'rE ret_valueall_gather_quantize_listall_gather_nonquantize_listrArArBrs@         zInit._all_gathercCsN|D]"}td|jdd|jdur|||j||ddtj|_qdS)NzBefore Partitioning Param FrTr)rhrr_partition_param_sec_partition_paramr0rr/)r@rrfrrr'rArArBrs   zInit._partitionc Cs|jtjusJd|dtd|jd|jdd|jtjurtd|jdtdd|jdur|std |jd|j dd|rKt |td |jd|j dd|jj t j krtd |jd dd|j|gtd |jd|jj d dddS||}||j}|jdurd}|jt j kr|jj|drt j }|j||}tjd|j|jd}|j|_td|jdn:|jr|j} n |jt j krt j} n|j} tj||j| d}|js|jr|j !|\}|_"| t jkr|j#rt$#|}d|_||_||j_%t&j|j_'||j_ ||_(||)} | |} |*+d} | |j%krU| |j%krU| ,d| |} t-|j.| Wdn 1sOwYn0| |j%kr|j%| }t-|j,dd|.| ,d| |Wdn 1swYtd |jd|j ddt |td |jd|j dd|jj t j kr|j/|gtd|jdtd|jdddtd|jd|jd|jd|j dSdS)N # Cannot partition a param in flight Param id z status is FrzPartitioning param id z reuse buffers zBefore partitioning param zAfter partitioning param zParam z+ partition released since it exists in nvmezafter swap Param )rsrrzID z< Initializing partition for the first time for nvme offload.z0 Offloaded to nvme offload and buffers released. partitioned type  dev  shape )0r/r0r1rhrrH reuse_buffersr5rrrrrrr$remove_partition_and_release_buffersrrrrswappable_tensor get_bufferr~rrr-r9r rrrrr7rxr+rrrWr!rrr contiguousr<rUno_gradrswap_out_and_release)r@r'bufferrr tensor_sizepartition_sizerpartitioned_tensorr-rend one_dim_param src_tensor elems_to_copyrArArBr+s              ,zInit._partition_paramc Csp|jtjusJd|d|jtjur|jdur|sdS||}||j}t||j}|jdurhd}t j ||j |j d}|j rF| }|jsU|jrU|j|\}|_d|_||_||j_tj|j_||j_||j} | |} |d} tdt|j| |} |jdd| | d| | tst !t"|j#d|j d|j$d |j%dd dSdS) Nr,r-rFr/rr0r1r2r)&r/r0r1rHrrrrdrr~rrrrrrr7rxr+rWr!rrrr7r<maxrYrUrrrErFrGrhrr-r) r@r'r:rr;r<secondary_partition_sizersecondary_partitioned_tensorsecondary_start secondary_endr? sec_numelrArArBr*sL          " zInit._partition_param_secc Cs~|jdur#td|jd|jd|jd|jd|j dStd|jd|jd|jd|jd|j dS)Nr.z, param status: z, param numel z, partitioned numel z , data numel z, partitioned ds_tensor )r5rhrr/rWrsr9r?rArArB _param_statuss 2.zInit._param_statusc Cs|jj}||j}||}||ks Jd|jd|d|td|dt|d|tdt|d|d d d tj ||j |j d  d }tdt|d |d |d d d t sht td|dt|d||jrtj||jt |||d}n=g} t|jD]%} | |d|| || tj||dkr| | jj|jjddqtj| | ||||d}|dd|j |j} | j|_|S)Nz param id z aligned size z does not match tensor size rz'---- Before allocating allgather param z partition size=z Before allocate allgather param z partition_size=r,Frrr/zAfter allocate allgather param z----allgather param with r[rrT)r.)r5rWrrrrhrrr~r3rr-r<rrErGrr^all_gather_into_tensorr:rget_partition_dp_grouprTrrUrbr9rrrr=) r@r'r]rr<r;aligned_param_sizer\rErNr]replicated_tensorrArArBr#sR  $    zInit._allgather_paramc Cs<t|dkrdS|jdkrt|}|dSg}g}|r!g}g}|D]-} || jj|| jt |rP|| jj || jj t q#g} |rWg} |D]} | |j} t j | |djj|jdd}d|_| |qY|r|D] } | |j} t j | |djj j|jdd}d|_| |q}g}g}t|D]\}} ||d}|jrtj| |||| dd}|rtj| ||||| dd}||nvg}t|jD]-}||} | |d|| | }||t|std |d |d |qtj|||| dd}|rPg}t|jD]}||} | |d|| | }||q%tj||||| dd}||||q|d|ri|D]}|qat|D]#\}} | |}|r|j|| |}|dd| j| j j!| _!qmt"st#dS) zO blocking call avoid explicit memory copy in _allgather_params rNr rr/FTr[zparam z , partition z! is not on CUDA, partition shape )$r}rr~rJrr5rWr:rrr+rsr~rrrr<r enumeraterr^rIrJrTrUrrrsizerr7r8r=r9rErG)r@rrrxrEpartition_sizes local_tensorsquantize_scale_sizesquantize_scale_tensorsr'allgather_paramsallgather_quantize_scalepsizer;r\launch_handleslaunch_quantize_handles param_idxrYhquantize_handle output_listr]routput_scale_listrGgathered_tensorrArArBr%s                  z Init._allgather_params_coalescedcCst|dkrdStdd|D}||j}tj||djj|jd}g}t|jD]3}||}| | d|||| kr]d} |D]} | jj } || d| |  | jj| | 7} qDq*t|ddrtdd|D} | |j} tj| |djjj|jd}g}t|jD]4}| |}| | d|| ||jkrd} |D]} | jjj }|| d| | | jjj| |7} qqtj||| || ddt|ddrtj||dj|| ddd}|D]a} | jj }| j }tj| j| jj|jd}t|jD]-}||}||}||kr-t|||}|| d||}|d  d|| |q|| jj 7}t|ddrD|j||}|j| _qdS) NrcSrrArrrArArBrkrz*Init._allgather_params..rr+cSsg|]}|jjqSrA)r5r+rsrrArArBrrFr[r/)r}rrr~rr5rrrTrrUrrWrr9r4rOr+rgr^rIrJrr=rYr<r7r8)r@rrr<r;r\rNr]roffsetr' param_numel scale_sizescale_tensor_sizeflat_scale_tensorscale_partitionsparam_scale_numelr^param_partition_size param_sizerLr` numel_to_copyrarArArBr$fs              zInit._allgather_paramsc Csg}|D] }|j|jksJ|jd|jd|||qt||D]A\}\}}|dur8||jj}||}||}||jkrQ|krknq*|j|} |j d d||  | dd| q*dS)Nz != zE Cannot reduce scatter gradients whose size is not same as the paramsr/r) r rsrWr_reduce_scatter_gradientziprJr5rr<rUr) r@rhandles_and_reduced_partitionsr'rEreduced_partitionr<rr>elementsrArArBrs&  &zInit._reduce_scatter_gradientsc Cs|jj}||j}g}t|jD]L}||}||}||jkr0||jkr0|jdd||}n'tj||j |j d}||jkrW|j|} |dd|  |jdd|| | |qt j||d} t j|| |||dd} | || fS)Nr/rrrTr[)r5rWrrTr r<rUr~r3rr-rrr^rbrJreduce_scatter) r@r'r< total_size input_listr]rr>inputrlrgrErArArBrhs(   &   zInit._reduce_scatter_gradientcCs>|dur dgt|}t||D] \}}|j|||dqdS)N)partition_bufferr)r}ri_partition_gradient)r@rrrr'rqrArArBrs zInit._partition_gradientsc Cs~td|jd|jd|jjd|jjtddd|jj}|dur7|r,Jdtj ||j|j d }n||ksIJd |d |t j | |d }||}||}|d dd|}||jkrt|j||} |dd| } |jd d|| } |s| | n'| j | j kr| | ntj| |j|j d } | | | | | | |j|j_tddddS)Nrz gradient of size z type z part_size zBefore partitioning gradientsFrzNo buffer to accumulate torzThe partition buffer size z* should match the size of param.ds_tensor rr/rzAfter partitioning gradients)rhrr rsrr5rWrr~r3r-r^rbrJr<rUrYradd_rr9) r@r'rqrr<rgrr>dest_tensor_full_bufferrl dest_tensorr@ acc_tensorrArArBrrs>*          zInit._partition_gradientcCs|jSrDrr?rArArBrJ6szInit.get_partition_dp_groupcCr)z]subclass can overload to specify different relative rank in parameter partition group)rgrmrArArBr9szInit.get_partition_rankcCrrD)rrmrArArBr>zInit.num_partitionscCr)z= Return the communication group with all data-parallel ranks rwrmrArArBrBrxzInit.get_dp_process_group)NNTNFNNTNNNFFNNrc)FFT)NFT)NF)Fr)rFr),rMrNrOrrrrrrrrr(rCrrrrrrrrrrrrrr+r*rHr#r%r~r8r$rrhrrrrJrpropertyrr __classcell__rArArrBr69st   V p  & y 4 6i O ! > r6c@s&eZdZd ddZddZddZdS) GatheredParametersNTcCs||_|sdSt|trt|tjst|}n|g}tdd|Ds(d|_dSdd|D|_tt |jddd |_d|_ |dur\|jd j t krQ||_ n t |jd j ||_ ||_|jdurp|jD] }t|j|qgdSdS) aA context that collects parameters that were partitioned via a :class:`deepspeed.zero.Init` context. The parameters are partitioned again upon exit. Args: params (``torch.nn.Parameter``): A single parameter, or an iterable of parameters (list, tuple, generator) of parameters to collect. It's assumed that all parameters are zero params. modifier_rank (int, optional): If specified, this rank's parameter will be broadcasted on exit from the context. This argument is required if ``params`` are modified, so that all processes have a consistent view of the data. Defaults to ``None``. fwd_module (``torch.nn.Module``, optional): If specified, ``params`` will be registered as external parameters of ``fwd_module``. See :meth:`deepspeed.zero.register_external_parameter`. enabled (bool, optional): If ``False``, this context is a no-op. Defaults to ``True``. Important: Make sure to use ``modifier_rank`` that is not ``None`` (e.g., ``modifier_rank=0``) if you need the GPU memory allocated by gather to be released upon exit from the context manager. Important: if ``params`` isn't an iterable of parameters or a single parameter it'll be silently ignored! Examples ======== #. Allocate a partitioned module, initialize its weight on rank 0, and update all processes. .. code-block:: python with deepspeed.zero.Init(): linear = torch.nn.Linear(1000,1000) with deepspeed.zero.GatheredParameters(linear.weight, modifier_rank=0): if deepspeed.comm.get_rank() == 0: linear.weight.zero_() with deepspeed.zero.GatheredParameters(linear.weight, modifier_rank=0): if deepspeed.comm.get_rank() == 0: linear.weight.zero_() #. Collect a partitioned weight to pass to another module during training. The parameter will be registered as an external parameter and made available during the backward pass. .. code-block:: python :emphasize-lines: 6 def forward(self, input): x = self.layer1(input) # self.layer1.weight is required by self.layer2.forward with deepspeed.zero.GatheredParameters(self.layer1.weight, fwd_module=self): y = self.layer2(x, self.layer1.weight) return y #. Pretrained model loading .. code-block:: python with deepspeed.zero.Init(): model = MyModel() state_dict = torch.load(model_path, map_location="cpu") def load(module: nn.Module, prefix=""): # because zero3 puts placeholders in model params, this context # manager gathers (unpartitions) the params of the current layer, then loads from # the state dict and then re-partitions them again with deepspeed.zero.GatheredParameters(list(module.parameters(recurse=False)), modifier_rank=0): if deepspeed.comm.get_rank() == 0: module._load_from_state_dict(state_dict, prefix) for name, child in module._modules.items(): if child is not None: load(child, prefix + name + ".") load(model, prefix="") If this approach is not used, then the full model will first be copied to each GPU. For models bigger than the memory of a single GPU, this method is required. Ncss|]}t|VqdSrDrrrArArBrrz.GatheredParameters.__init__..FcSsg|] }t|dr|qSr)r4rrArArBrsz/GatheredParameters.__init__..cSrrDr)xrArArBrrz-GatheredParameters.__init__..rr)rr}rr~rranyrRrrsrc_rankrr^rr fwd_moduler)r@rR modifier_rankrrrrArArBrCIs0V   zGatheredParameters.__init__cCs"|jsdS|jdj|jddS)Nrr)rrRrrmrArArBrszGatheredParameters.__enter__csnjsdSjdurjdjjdddSfddjD}|D]}|q#jdjjdddS)NrF)rrcs$g|]}tj|jj|jddqS)Tr[)r^rr9r~rrrmrArBrs$z/GatheredParameters.__exit__..T)rr~rRrrJ)r@excrgrYrArmrBrs  zGatheredParameters.__exit__)NNT)rMrNrOrCrrrArArArBr{Gs w r{rDrrL)mrurrwtypingrrenumrrrrr collectionsrrr~rrlr r^torch.nnr r r+r deepspeed.utilsrutilsrrdeepspeed.runtime.zero.configrdeepspeed.runtime.zero.utilsrr%deepspeed.runtime.zero.offload_configrdeepspeed.runtime.config_utilsrrrdeepspeed.comm.commrdeepspeed.utils.debugrrrrrdeepspeed.acceleratorr%swap_tensor.partitioned_param_swapperr r!&deepspeed.inference.quantization.utilsr"r#r$r%partitioned_param_data_shaperrr&rQr`rhstrrjr{rrrr0rr1rrr3r2r5r4r7r6r9r8r;r:r=r<rrrrrr3temp_contiguous_tensor empty_buffersobjectrr?r@rArMrfrhrirjr~r6r{rArArArBs                4    < J  )