o
    Ti	8                     @   s   d dl Z d dlmZmZmZ d dlmZ d dlmZm	Z	 d dl
mZmZmZ d dlmZ ddlmZmZmZ d	Zd
Zdd Zdd ZG dd deeZG dd deZdS )    N)OptionalDictAny)Enum)Fieldmodel_validator)get_scalar_parampp_intDeepSpeedConfigModel)logger   )DeepSpeedZeroOffloadParamConfig#DeepSpeedZeroOffloadOptimizerConfigOffloadDeviceEnuma:  
ZeRO optimization should be enabled as:
"session_params": {
  "zero_optimization": {
    "stage": [0|1|2],
    "stage3_max_live_parameters" : 1000000000,
    "stage3_max_reuse_distance" : 1000000000,
    "stage3_use_all_reduce_for_fetch_params": [true|false],
    "stage3_module_granularity_threshold": 0,
    "allgather_partitions": [true|false],
    "use_multi_rank_bucket_allreduce": [true|false],
    "allgather_bucket_size": 500000000,
    "reduce_scatter": [true|false],
    "contiguous_gradients" : [true|false]
    "overlap_comm": [true|false],
    "reduce_bucket_size": 500000000,
    "load_from_fp32_weights": [true|false],
    "cpu_offload": [true|false] (deprecated),
    "cpu_offload_param" : [true|false] (deprecated),
    "cpu_offload_use_pin_memory": [true|false] (deprecated),
    "sub_group_size" : 1000000000000,
    "offload_param": {...},
    "offload_optimizer": {...},
    "ignore_unused_parameters": [true|false],
    "round_robin_gradients": [true|false],
    "zero_hpz_partition_size": 1,
    "zero_quantized_weights": [true|false],
    "zero_quantized_nontrainable_weights": [true|false],
    "zero_quantized_gradients": [true|false],
    "memory_efficient_linear": [true|false],
    "override_module_apply": [true|false],
    "zeropp_loco_param": {...},
    "log_trace_cache_warnings" : [true|false],
    }
}
zero_optimizationc                 C   sH   i }| t  rdnd|d< |d dkrt| dd|d< tdt |S )Nr   r   stageallgather_size    eͽAallgather_bucket_sizezjDeepSpeedConfig: this format of ZeRO optimization setup is deprecated. Please use the following format: {})ZERO_OPTIMIZATIONr   r   warningformatZERO_FORMAT
param_dictzero_config_dict r   Q/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/zero/config.pyread_zero_config_deprecated8   s   r   c                 C   s6   t | v r| t  }t|trt| }ni }tdi |S )Nr   )r   
isinstanceboolr   DeepSpeedZeroConfigr   r   r   r   get_zero_configC   s   
r"   c                   @   s$   e Zd ZdZdZdZdZdZdZdS )ZeroStageEnumz% Enum class for possible zero stages r   r         N)	__name__
__module____qualname____doc__disabledoptimizer_states	gradientsweights	max_stager   r   r   r   r#   M   s    r#   c                   @   s\  e Zd ZU dZdZeed< 	 dZeed< 	 dZ	eed< 	 e
edddZeed	< 	 dZeed
< 	 dZeed< 	 e
edddZeed< 	 dZee ed< 	 dZeed< 	 dZeed< 	 dZee ed< 	 dZee ed< 	 e
edddZeed< 	 e
ddddd ddZee ed< 	 e
ddddddZee ed< 	 e
ddddd ddZee ed< 	 e
ed dd!d"Zeed#< 	 e
ed$dd%d"Zeed&< 	 e
eej d'dd(d"Z!eed)< 	 e
eddd*d"Z"eed+< 	 e
eddd,d"Z#eed-< 	 e
dd.d/Z$eed0< 	 e
edd1d/Z%eed2< 	 e
dd3d/Z&eed4< 	 e
ddd0d5dZ'eed6< 	 dZ(eed7< 	 dZ)eed8< 	 dZ*eed9< 	 e
d:ddZ+eed;< 	 dZ,eed<< 	 dZ-eed=< 	 dZ.eed>< 	 dZ/ee0e1e2f  ed?< 	 e
d@dAdBidZ3eedB< dZ4eedC< dZ5eedD< 	 	 dZ6eedE< dZ7eedF< 	 dZ8eedG< 	 e9dHdIdJdK Z:e9dHdIdLdM Z;dS )Nr!   z1
    Sets parameters for ZeRO optimizations.
    r   r   Tcontiguous_gradientsreduce_scatterr   )gereduce_bucket_sizeuse_multi_rank_bucket_allreduceallgather_partitionsr   Noverlap_commload_from_fp32_weightsFelastic_checkpointoffload_paramoffload_optimizerg    eAsub_group_sizec                 C      | rt tjdS d S N)device)r   r   cpuvalr   r   r   <lambda>      zDeepSpeedZeroConfig.<lambda>)
deprecated	new_paramnew_param_fn)json_schema_extracpu_offload_paramz"offload_param or offload_optimizer)rC   rD   set_new_paramcpu_offload_use_pin_memoryc                 C   r;   r<   )r   r   r>   r?   r   r   r   rA      rB   cpu_offloadg    ׇAstage3_prefetch_bucket_size)r1   aliasprefetch_bucket_sizeg     j@"stage3_param_persistence_thresholdparam_persistence_thresholdzsys.maxsize"stage3_model_persistence_thresholdmodel_persistence_thresholdstage3_max_live_parametersmax_live_parametersstage3_max_reuse_distancemax_reuse_distance)stage3_gather_16bit_weights_on_model_save)rL   "gather_16bit_weights_on_model_save#stage3_module_granularity_thresholdmodule_granularity_threshold&stage3_use_all_reduce_for_fetch_paramsuse_all_reduce_for_fetch_params)rC   rD   (stage3_gather_fp16_weights_on_model_saveignore_unused_parameterslegacy_stage1round_robin_gradientsr   zero_hpz_partition_sizezero_quantized_weights#zero_quantized_nontrainable_weightszero_quantized_gradientszeropp_loco_paramrD   mics_shard_sizemics_hierarchical_params_gathermemory_efficient_linearpipeline_loading_checkpointoverride_module_applylog_trace_cache_warningsafter)modec                 C   s   | j d u r| jtjk| _ | S )N)r5   r   r#   r-   )selfr   r   r   overlap_comm_valid^  s   
z&DeepSpeedZeroConfig.overlap_comm_validc                 C   s,   | j }|r|jdk r| jtjksJ d| S )Ng      ?z3Partial offloading only supported for ZeRO Stage 3.)r9   ratior   r#   r-   )rn   offload_configr   r   r   offload_ratio_checkd  s   z'DeepSpeedZeroConfig.offload_ratio_check)<r&   r'   r(   r)   r   r#   __annotations__r/   r    r0   r   r	   r2   intr3   r4   r   r5   r   r6   r7   r8   r   r9   r   r:   rG   rI   rJ   rM   rO   sysmaxsizerQ   rS   rU   rW   rY   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   r   strr   rf   rg   rh   ri   rj   rk   r   ro   rr   r   r   r   r   r!   V   s   
 	

r!   )ru   typingr   r   r   enumr   pydanticr   r   deepspeed.runtime.config_utilsr   r	   r
   deepspeed.utilsr   rq   r   r   r   r   r   r   r"   rt   r#   r!   r   r   r   r   <module>   s   %
	