o
    Tia                     @   s   d dl T ddlZddlmZmZ dd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 ZdS )4   )*    N   )get_scalar_paramget_list_paramc                 C   sx   i }t |  vri | t < | t  }t||t< t||t< t||t< t||t	< t
||t< t||t< t||t< |S N)COMPRESSION_TRAININGkeysget_weight_quantizationWEIGHT_QUANTIZATIONget_activation_quantizationACTIVATION_QUANTIZATIONget_sparse_pruningSPARSE_PRUNINGget_row_pruningROW_PRUNINGget_head_pruningHEAD_PRUNINGget_channel_pruningCHANNEL_PRUNINGget_layer_reductionLAYER_REDUCTION
param_dictoutputsub_param_dict r   P/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/compression/config.pyget_compression_config   s   r   c                 C   sB   i }t |t< t| rt| |t< t|  D ]\}}|||< q|S r   )LAYER_REDUCTION_ENABLED_DEFAULTLAYER_REDUCTION_ENABLEDget_layer_reduction_enabledget_layer_reduction_paramsitems)r   r   keyvalr   r   r   r      s   
r   c                 C   s    t |  v rt| t  ttS dS NF)r   r	   r   r    r   )r   r   r   r   r!   (   s   r!   c                 C   s,   t |  v rt| t  }|t |S dS r&   )r   r	   copypopr    )r   layer_reduction_paramsr   r   r   r"   /   s
   
r"   c                 C   s(   t |  vrdS | t  }t|}|t S r&   )r   r	   )get_weight_quantization_shared_parametersWEIGHT_QUANTIZE_ENABLED)r   r   r   r   r   r   get_quantize_enabled8   s
   r,   c                 C   l   i }t |  vrti ti i| t < | t  }t||t< |t t r.t| v s.J dt dt||t< |S )NzWeigh Quantization is enabled,  must be specified)r   r	   SHARED_PARAMETERSDIFFERENT_GROUPSr*   r+   (get_weight_quantization_different_groupsr   r   r   r   r
   A      
r
   c                 C   s|  i }t |  v r| t  }t|tt|t< t|tt|t< t|tt|t< t|t	t
|t	< t|tt|t< t|tt|t< t|tt|t< |t ttfv sWJ dt dt dt|tt|t< |t ttfv srJ dt dt dt| v rt|t tt|t< t|t tt|t< |S t|t< t|t< |S t|t< t|t< t|t< t
|t	< t|t< t|t< t|t< t|t< t|t< |S )Nz0Invalid weight quantize type. Supported types: [, ]z4Invalid weight quantize rounding. Supported types: [)r/   r	   r   r+   WEIGHT_QUANTIZE_ENABLED_DEFAULTWEIGHT_QUANTIZE_KERNELWEIGHT_QUANTIZE_KERNEL_DEFAULTWEIGHT_QUANTIZE_SCHEDULE_OFFSET'WEIGHT_QUANTIZE_SCHEDULE_OFFSET_DEFAULTWEIGHT_QUANTIZE_GROUPSWEIGHT_QUANTIZE_GROUPS_DEFAULTWEIGHT_QUANTIZE_VERBOSEWEIGHT_QUANTIZE_VERBOSE_DEFAULTWEIGHT_QUANTIZE_TYPEWEIGHT_QUANTIZE_TYPE_DEFAULT"WEIGHT_QUANTIZE_IN_FORWARD_ENABLED*WEIGHT_QUANTIZE_IN_FORWARD_ENABLED_DEFAULTWEIGHT_QUANTIZE_SYMMETRICWEIGHT_QUANTIZE_ASYMMETRICWEIGHT_QUANTIZE_ROUNDING WEIGHT_QUANTIZE_ROUNDING_DEFAULT WEIGHT_QUANTIZE_NEAREST_ROUNDING#WEIGHT_QUANTIZE_STOCHASTIC_ROUNDING#WEIGHT_QUANTIZE_FP16_MIXED_QUANTIZE+WEIGHT_QUANTIZE_FP16_MIXED_QUANTIZE_ENABLED3WEIGHT_QUANTIZE_FP16_MIXED_QUANTIZE_ENABLED_DEFAULTWEIGHT_QUANTIZE_CHANGE_RATIO$WEIGHT_QUANTIZE_CHANGE_RATIO_DEFAULTr   r   r   r   r*   P   sz   r*   c                 C   |   i }| t  }dd }| D ]-\}}i ||< |||| t || t< t|| tt|| t< t|| tt|| t< q|S )Nc                 S   sP   t | v sJ t  d|  t| v sJ t d|  t|tt|t< |S )Nz1 must be specified for weight quantization group )WEIGHT_QUANTIZE_START_BITSr	   WEIGHT_QUANTIZE_TARGET_BITSr   WEIGHT_QUANTIZATION_PERIOD"WEIGHT_QUANTIZATION_PERIOD_DEFAULTname
group_dictr   r   r   
get_params   s   z<get_weight_quantization_different_groups.<locals>.get_paramsr0   r#   DIFFERENT_GROUPS_PARAMETERSr   DIFFERENT_GROUPS_MODULE_SCOPE%DIFFERENT_GROUPS_MODULE_SCOPE_DEFAULT%DIFFERENT_GROUPS_RELATED_MODULE_SCOPE-DIFFERENT_GROUPS_RELATED_MODULE_SCOPE_DEFAULTr   r   r   rU   kvr   r   r   r1      s   	

r1   c                 C   r-   )Nz$Activation Quantization is enabled, r.   )r   r	   r/   r0   -get_activation_quantization_shared_parametersACTIVATION_QUANTIZATION_ENABLED,get_activation_quantization_different_groupsr   r   r   r   r      r2   r   c                 C   s   i }t |  v rT| t  }t|tt|t< t|tt|t< |t ttfv s/J dt dt dt|t	t
|t	< |t	 ttfv sJJ dt dt dt|tt|t< |S t|t< t|t< t
|t	< t|t< |S )Nz4Invalid activation quantize type. Supported types: [r3   r4   zAInvalid activation quantize range calibration. Supported types: [)r/   r	   r   r`   'ACTIVATION_QUANTIZATION_ENABLED_DEFAULTACTIVATION_QUANTIZE_TYPE ACTIVATION_QUANTIZE_TYPE_DEFAULTACTIVATION_QUANTIZE_SYMMETRICACTIVATION_QUANTIZE_ASYMMETRICACTIVATION_QUANTIZE_RANGE!ACTIVATION_QUANTIZE_RANGE_DEFAULT!ACTIVATION_QUANTIZE_RANGE_DYNAMIC ACTIVATION_QUANTIZE_RANGE_STATIC#ACTIVATION_QUANTIZE_SCHEDULE_OFFSET+ACTIVATION_QUANTIZE_SCHEDULE_OFFSET_DEFAULTr   r   r   r   r_      s@   r_   c                 C   rM   )Nc                 S   "   t | v sJ t  d|  |S )Nz5 must be specified for activation quantization group )ACTIVATION_QUANTIZE_BITSr	   rR   r   r   r   rU         z@get_activation_quantization_different_groups.<locals>.get_paramsrV   r\   r   r   r   ra         

ra   c                 C   s|   i }t |  vrti ti i| t < | t  }t||t< |t t r6|t t tkr6t| v s6J dt dt||t< |S )Nz8Sparse Pruning is enabled and not snip_momentum method, r.   )	r   r	   r/   r0   $get_sparse_pruning_shared_parametersSPARSE_PRUNING_ENABLEDSPARSE_PRUNING_METHOD#SPARSE_PRUNING_METHOD_SNIP_MOMENTUM#get_sparse_pruning_different_groupsr   r   r   r   r      s   
r   c                 C   s.  i }t |  v r| t  }t|tt|t< t|tt|t< |t ttt	fv s3J dt dt dt	 dt|t
t|t
< |t t	krt|tt|t< t|tt|t< |t dkr]|t dk saJ dt|tt|t< t|tt|t< t|t|t
 |t< |t
 |t ksJ d|S t|t< t|t< t|t
< |S )Nz1Invalid sparse pruning method. Supported types: [r3   r4   r   r   z.Invalid dense_ratio value. Must be less than 1z6Invalid schedule_offset and schedule_offset_end values)r/   r	   r   rr   SPARSE_PRUNING_ENABLED_DEFAULTrs   SPARSE_PRUNING_METHOD_DEFAULTSPARSE_PRUNING_METHOD_L1SPARSE_PRUNING_METHOD_TOPKrt   SPARSE_PRUNING_SCHEDULE_OFFSET&SPARSE_PRUNING_SCHEDULE_OFFSET_DEFAULTSPARSE_PRUNING_BLOCK_PATTERN$SPARSE_PRUNING_BLOCK_PATTERN_DEFAULTSPARSE_PRUNING_DENSE_RATIO"SPARSE_PRUNING_DENSE_RATIO_DEFAULT%SPARSE_PRUNING_SCHEDULE_OFFSET_STRIDE-SPARSE_PRUNING_SCHEDULE_OFFSET_STRIDE_DEFAULTr   SPARSE_PRUNING_EXCLUDED_MODULES'SPARSE_PRUNING_EXCLUDED_MODULES_DEFAULT"SPARSE_PRUNING_SCHEDULE_OFFSET_ENDr   r   r   r   rq      sf   rq   c                 C   rM   )Nc                 S   rm   )Nz, must be specified for sparse pruning group )r~   r	   rR   r   r   r   rU     ro   z7get_sparse_pruning_different_groups.<locals>.get_paramsrV   r\   r   r   r   ru     rp   ru   c                 C   r-   )NzRow Pruning is enabled, r.   )r   r	   r/   r0   !get_row_pruning_shared_parametersROW_PRUNING_ENABLED get_row_pruning_different_groupsr   r   r   r   r   !  r2   r   c                 C      i }t |  v r9| t  }t|tt|t< t|tt|t< |t ttfv s/J dt dt dt|t	t
|t	< |S t|t< t|t< t
|t	< |S )Nz.Invalid row pruning method. Supported types: [r3   r4   )r/   r	   r   r   ROW_PRUNING_ENABLED_DEFAULTROW_PRUNING_METHODROW_PRUNING_METHOD_DEFAULTROW_PRUNING_METHOD_L1ROW_PRUNING_METHOD_TOPKROW_PRUNING_SCHEDULE_OFFSET#ROW_PRUNING_SCHEDULE_OFFSET_DEFAULTr   r   r   r   r   0  s(   r   c                 C   rM   )Nc                 S   rm   )Nz) must be specified for row pruning group )ROW_PRUNING_DENSE_RATIOr	   rR   r   r   r   rU   G  ro   z4get_row_pruning_different_groups.<locals>.get_paramsrV   r\   r   r   r   r   C     

r   c                 C   r-   )NzHead Pruning is enabled, r.   )r   r	   r/   r0   "get_head_pruning_shared_parametersHEAD_PRUNING_ENABLED!get_head_pruning_different_groupsr   r   r   r   r   V  r2   r   c                 C   s   i }t |  v rP| t  }t|tt|t< t|tt|t< |t ttfv s/J dt dt dt|t	t
|t	< |t rNt| v sHJ t d|t |t< |S t|t< t|t< t
|t	< |S )Nz/Invalid head pruning method. Supported types: [r3   r4   z# must be specified for head pruning)r/   r	   r   r   HEAD_PRUNING_ENABLED_DEFAULTHEAD_PRUNING_METHODHEAD_PRUNING_METHOD_DEFAULTHEAD_PRUNING_METHOD_L1HEAD_PRUNING_METHOD_TOPKHEAD_PRUNING_SCHEDULE_OFFSET$HEAD_PRUNING_SCHEDULE_OFFSET_DEFAULTHEAD_PRUNING_NUM_HEADSr   r   r   r   r   e  s6   r   c                 C   rM   )Nc                 S   s   t | v sJ d|  |S )Nz5dense_ratio must be specified for head pruning group )HEAD_PRUNING_DENSE_RATIOr	   rR   r   r   r   rU     s   z5get_head_pruning_different_groups.<locals>.get_paramsrV   r\   r   r   r   r   }  r   r   c                 C   r-   )NzSparse Pruning is enabled, r.   )r   r	   r/   r0   %get_channel_pruning_shared_parametersCHANNEL_PRUNING_ENABLED$get_channel_pruning_different_groupsr   r   r   r   r     r2   r   c                 C   r   )Nz2Invalid channel pruning method. Supported types: [r3   r4   )r/   r	   r   r   CHANNEL_PRUNING_ENABLED_DEFAULTCHANNEL_PRUNING_METHODCHANNEL_PRUNING_METHOD_DEFAULTCHANNEL_PRUNING_METHOD_L1CHANNEL_PRUNING_METHOD_TOPKCHANNEL_PRUNING_SCHEDULE_OFFSET'CHANNEL_PRUNING_SCHEDULE_OFFSET_DEFAULTr   r   r   r   r     s,   r   c                 C   rM   )Nc                 S   rm   )Nz- must be specified for channel pruning group )CHANNEL_PRUNING_DENSE_RATIOr	   rR   r   r   r   rU     ro   z8get_channel_pruning_different_groups.<locals>.get_paramsrV   r\   r   r   r   r     rp   r   )	constantsr'   runtime.config_utilsr   r   r   r   r!   r"   r,   r
   r*   r1   r   r_   ra   r   rq   ru   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   s4   
		2%