o TiB@s ddlZddlmZddlmZddlmZmZm Z eGdddZ eGdddZ d d Z d1d d Z d1d dZddZddZddZddZddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*Zd+eejjd,e fd-d.Zd/d0Z dS)2N) dataclass)comm)DictListCallablec@seZdZUeed<eed<dS)fragment_addressnumelstartN)__name__ __module__ __qualname__int__annotations__rrS/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/utils/tensor_fragment.pyr s  rc@seZdZUejed<eed<ejed<eed<eed<eed<eed<e ed<d Z eed <d d Z d dZ ddZ ddZddZddZdddZddZd S)tensor_fragment lp_fragmentlp_fragment_address hp_fragmenthp_fragment_address gradient_dictoffload_gradient_dict use_offloadparam_group_indexNoptim_fragmentcC|jj|jjdSN)rdatacopy_rselfrrr update_hpztensor_fragment.update_hpcCrr)rrrrrrrr update_lp!r"ztensor_fragment.update_lpcCs"||jvr |j|St|d)Nz& not found in optimizer state fragment)r ValueError)r keyrrrget_optim_state_fragment$s  z(tensor_fragment.get_optim_state_fragmentcsfdd|D_dS)Ncs>i|]\}}t|r|jjkr||djjjjqS)r)torch is_tensorshapenarrowrr r).0r%valueflat_hp_partitionr rr +s z.)itemsr)r r.rrr-rset_optim_state_fragment*s  z(tensor_fragment.set_optim_state_fragmentcCs|jSr)rrrrrget_hp_fragment_address1sz'tensor_fragment.get_hp_fragment_addresscCst|jSr)listrkeysrrrrget_optim_state_keys4sz$tensor_fragment.get_optim_state_keyscCs|dur|jS||Sr)rr&)r optim_state_keyrrrget_hp_fragment7s ztensor_fragment.get_hp_fragmentcCsB|jr|j}n|j}|j|vs||jdurtd||j|S)NzNGradients are only available immediately after backward and before engine step)rrrrr$)r index_in_param_grouprrrrget_lp_grad_fragment<s z$tensor_fragment.get_lp_grad_fragmentr)r r r r'Tensorrrrboolr rr!r#r&r1r2r5r7r9rrrrrs$     rc Csx|D]7}|}t|}|D]%}|jdur2|j}|d|j|j} | j|jj |dj| |j_ q ||||<qdS)Nr)r6) r' zeros_like _hp_mappingr2r*r rrrr7r) flat_hp_tensor lp_tensors optim_stateopt_keysr%hp_parambufferlprrrrrmap_to_flat_opt_statesHs   rEcCsntj|tjd}|jdur*|jj}t|d|j|j}|j |}|j |j t j ||jd||S)Ndtypergroup)r'r<float32flattenr=rr*r rr7rrdist all_reduce _dp_group reshape_as)r r6 reduce_bufferlp_frag_addressreduce_fragmentrrrrget_full_hp_paramWs   rScCsL|jdur$|jj}t|d|j|j}|j|}|j |jdSdSNr) r=rr'r*rKr rr7rr)r r,r6rQvalue_fragmentrrrrset_full_hp_parambs  rVcCstj|tjd}|jdurD|j|j}|tj}|jj}t |d|j |j }| dj |j kr=|j|jn|j|jtj||jd||S)NrFrrH)r'r<rJrKr=r9_index_in_param_grouptorr*r rviewr)rrrLrMrNrO)r rPlp_grad_fragmenthp_grad_fragmentrQrRrrrget_full_hp_gradjs  r]cCsV|jdur)|j|j}|jj}t|d|j|j}|j |j |j dSdSrT) r=r9rXrr'r*rKr rrrrO)r r,r[rQrUrrrset_full_hp_grad|s r^cCs,t|dr |j|St|dr|SdS)zAssemble and return the fp32 parameter of a low-precision (e.g., fp16) parameter. Args: param (``torch.nn.Parameter``): A model parameter Returns: Union[torch.Tensor, None]: A tensor on accelerator device ds_idr=Nhasattr _z3_optimizerrSparamrrrsafe_get_full_fp32_params  recCs4t|dr |j||t|dr||dSdS)zUpdate the partitioned fp32 parameter of a low-precision (e.g., fp16) parameter. Args: param (``torch.nn.Parameter``): A model parameter value (``torch.Tensor``): New value r_r=NrarbrVrdr,rrrsafe_set_full_fp32_params  rhcCs0t|dr |j||St|dr||SdS)ahAssemble and return the fp32 optimizer state of a low-precision (e.g., fp16) parameter. Args: param (``torch.nn.Parameter``): A model parameter optim_state_key (``string``): Key value of optimizer state (e.g., `exp_avg` in Adam optimizer) Returns: Union[torch.Tensor, None]: A tensor on accelerator device r_r=Nr`rdr6rrrsafe_get_full_optimizer_states   rjcCs8t|dr |j|||t|dr|||dSdS)aCUpdate the partitioned fp32 optimizer state of a low-precision (e.g., fp16) parameter. Args: param (``torch.nn.Parameter``): A model parameter value (``torch.Tensor``): New value optim_state_key (``string``): Key value of optimizer state (e.g., `exp_avg` in Adam optimizer) r_r=Nrfrdr,r6rrrsafe_set_full_optimizer_states  rlcCs<|jdur|jSt|dr|j|St|dr|SdS)a Assemble and return the fp32 gradient of a low-precision (e.g., fp16) parameter. The return data type is that used for gradient accumulation. This is usually the param data type, but could also be different (e.g., bf16 param training with fp32 gradient accumulation). Args: param (``torch.nn.Parameter``): A model parameter Returns: Union[torch.Tensor, None]: A tensor on accelerator device Nr_r=)gradrarbget_fp32_grad_for_paramr]rcrrrsafe_get_full_grads    rocCsR|jdur |j|dSt|dr|j||dSt|dr'||dSdS)ab Update the partitioned gradient of a low-precision (e.g., fp16) parameter. To avoid precision issues, the update value should have the data type of gradient accumulation. Args: param (``torch.nn.Parameter``): A model parameter value (``torch.Tensor``): The un-partitioned new gradient value. Nr_r=)rmrrarbset_fp32_grad_for_paramr^rgrrrsafe_set_full_grads   rqcCt|ds Jd|j|S)a Get the local gradient partition of a ZeRO-3 partitioned parameter. The return data type is that used for gradient accumulation. This is usually the param data type, but could also be different (e.g., bf16 param training with fp32 gradient accumulation). Args: param (``torch.nn.Parameter``): A model parameter Returns: Union[torch.Tensor, None]: A tensor on accelerator device r_:This API is only defined for ZeRO-3 partitioned parameters)rarbget_local_fp32_grad_for_paramrcrrrsafe_get_local_grads rucC$t|ds Jd|j||dS)a_ Update the local gradient partition of a ZeRO-3 partitioned parameter. To avoid precision issues, the update value should have the data type of gradient accumulation. Args: param (``torch.nn.Parameter``): A model parameter. value (``torch.Tensor``): New value of local gradient partition. r_rsN)rarbset_local_grad_for_paramrgrrrsafe_set_local_grad s rxcCrr)zGet the local partition of a ZeRO-3 partitioned parameter in fp32 precision. Args: param (``torch.nn.Parameter``): A model parameter. Returns: Union[torch.Tensor, None]: A tensor on accelerator device r_rsrarbget_local_fp32_paramrcrrrsafe_get_local_fp32_params r{cCs t|ds Jd|j||S)aoGet the local optimizer state partition of ZeRO-3 partitioned parameter in fp32 precision. Args: param (``torch.nn.Parameter``): A model parameter optim_state_key (``string``): Key value of optimizer state (e.g., `exp_avg` in Adam optimizer) Returns: Union[torch.Tensor, None]: A tensor on accelerator device r_rsryrirrrsafe_get_local_optimizer_state's r|cCs&t|ds Jd|j|||dS)a`Update the local optimizer state partition of a ZeRO-3 partitioned parameter. Args: param (``torch.nn.Parameter``): A model parameter. value (``torch.Tensor``): New value of local optimizer state partition. optim_state_key (``string``): Key value of optimizer state (e.g., `exp_avg` in Adam optimizer). r_rsNrarbset_local_hp_paramrkrrrsafe_set_local_optimizer_state5srcCrv)zUpdate the local partition of ZeRO-3 partitioned parameter. Args: param (``torch.nn.Parameter``): A model parameter. value (``torch.Tensor``): New value of local parameter partition. r_rsNr}rgrrrsafe_set_local_fp32_paramAsr param_list update_funccCsg}|D]}|jdur|j||j|q|jr||q|s#dSt|ddr5|dj||dSt|ddrO|D]}t|}|||}||q>dSdS)a Vectorized update of the partitioned gradients of a list of low-precision (e.g., fp16) parameters. To avoid precision issues, the update value should have the data type of gradient accumulation. Args: param_list (``List[torch.nn.Parameter]``): List of model parameters update_func (``torch.Tensor``): A function that takes current full gradient value and returns new one. Nrr_r=) rmr requires_gradappendrarb%update_fp32_grad_for_param_vectorizedror^)rrpartitioned_grad_paramspold_gradnew_gradrrr safe_update_full_grad_vectorizedPs(      rc  Cs||} |} ||} t|| } t| | } | | ks$Jd| d| | | }t| | |d}|d|j|j}t| ||d}|d|j|j}t||||||||dS)Nzfragment start z should be < fragment_end )r rr)rrrrrrrr)rmaxminrr*r rKr)lp_paramlp_startr.rrrrpartition_startpartition_sizelp_endhp_starthp_endfragment_start fragment_endfragment_numelhp_frag_addresshp_fragment_tensorrQlp_fragment_tensorrrrget_hp_fragment_mappingss,    rr)!r' dataclassesr deepspeedrrLtypingrrrrrrErSrVr]r^rerhrjrlrorqrurxr{r|rrnn Parameterrrrrrrs8  5    #