o
    Tin.                  
   @   s   d dl Z ddlmZmZmZmZ ddlmZ ddlm	Z	 ddl
T d dlZd dlZzd dlZW n eyB Z zdZW Y dZ[ndZ[ww dd	 ZdddZdddZdddZdddZdd ZdS )    N   )compression_preparationfix_compressionrecursive_getattris_module_compressible)get_compression_config   )"dict_raise_error_on_duplicate_keys)*c                 C   s<   t | tr| S tj| rtjt| dtdS t	d|  )Nr)object_pairs_hookzSExpected a string path to an existing deepspeed config, or a dictionary. Received: )

isinstancedictospathexistsjsonloadopenr	   
ValueError)config r   R/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/compression/compress.pycheck_deepspeed_config   s   
r   Tc           
      C   sx   g }|  D ]1\}}t||}	t||dur7|	r7||v r)|r)t| d|  d||vr7|| || q||fS )z_
    get the associated module name from the model based on the key_word provided by users
    NzD is already added to compression, please check your config file for .)named_modulesr   researchr   addappend)

group_namemodelkey_wordexist_module_namempuverbosereturn_module_namenamemodulemodule_checkr   r   r   get_module_name   s   


r*   c                 C   s0  g }|  D ]\}}t|v rqt }|t }|t   D ]x\}}	g }
g }|	t rdt|	t |	t D ]1\}}t|| |||d\}}|
	| g }|D ]}t|| |t |d\}}|	| qI|	| q1n|	t D ]}t|| |||d\}}|
	| qh|
ri |	
 t|}|
|||ig}|	| qq|S )Nr$   )itemsLAYER_REDUCTIONsetSHARED_PARAMETERSDIFFERENT_GROUPS%DIFFERENT_GROUPS_RELATED_MODULE_SCOPEzipDIFFERENT_GROUPS_MODULE_SCOPEr*   r   copypopDIFFERENT_GROUPS_PARAMETERS)r!   compress_methodsr$   layer_added_compress_methodsmethodmethod_contentr#   shared_parametersr    method_parametersmodule_name_listrelated_module_name_listr"   related_key_wordsmodule_nametmp_related_module_name_listrkw_combined_method_parameterscompression_itemr   r   r   get_compress_methods2   sZ   



&rF   c                 C   s   t t|}t| dr| j}n| }|t t r%|dusJ dt||| t|||d}t||| |t	 t
 }|t r||t tkr|tdusJJ dddlm}m}	 dd	lm}
 |
d|t  |t |t |t |t |t d
}|||}||_|	| | S )a@  
    Compress a model: replace linear/conv2d layer with deepspeed compression-aware modules
    Args:
        model (`torch.nn.Module`)
            The model to compress.
        deepspeed_config (`DeepSpeedConfig`)
            The path of ds_config
        mpu
            The mpu module for Row/Column parallelism
    r(   Nz-Teacher model is required for layer reductionr+   zplease ensure the neural_compressor python package is installed by pip or conda if user wants to use snip_momentum sparse pruningr   )generate_prunersregister_on_step_beginr   )WeightPruningConfig)target_sparsitypatternpruning_frequency
start_stepend_stepexcluded_op_names)r   r   hasattrr(   r-   LAYER_REDUCTION_ENABLEDstudent_initializationrF   r   SPARSE_PRUNINGr/   SPARSE_PRUNING_ENABLEDSPARSE_PRUNING_METHOD#SPARSE_PRUNING_METHOD_SNIP_MOMENTUMnchelperrG   rH   rI   SPARSE_PRUNING_DENSE_RATIOSPARSE_PRUNING_BLOCK_PATTERN%SPARSE_PRUNING_SCHEDULE_OFFSET_STRIDESPARSE_PRUNING_SCHEDULE_OFFSET"SPARSE_PRUNING_SCHEDULE_OFFSET_ENDSPARSE_PRUNING_EXCLUDED_MODULESpruners)r!   deepspeed_configteacher_modelr$   r7   c_modelr8   r;   rG   rH   rI   r   r_   r   r   r   init_compressiond   s4   

rc   c                    s   t t|}t| dr| j}n| }t|||d}ttttt	t
g t| fddd}|D ]G\}}}	g }
|r7dnd}t|D ]5\}}|D ]}t|||	|d}|rT|
| qC|rr|| D ]}t|D ]\}}t|||	|
| dd	}qaq[q=q,| S )
a
  
    Remove the redundancy of a model
    Args:
        model (`torch.nn.Module`)
            The model to compress.
        deepspeed_config (`DeepSpeedConfig`)
            The path of ds_config
        mpu
            The mpu module for Row/Column parallelism
    r(   r+   c                    s     t| d  d S )Nr   r   )indexlistkeys)x
order_listr   r   <lambda>   s    z"redundancy_clean.<locals>.<lambda>)keyTF)dim_reduction)maskrl   )r   r   rP   r(   rF   WEIGHT_QUANTIZATIONrS   ROW_PRUNINGHEAD_PRUNINGCHANNEL_PRUNINGACTIVATION_QUANTIZATIONsorted	enumerater   r   )r!   r`   r$   r7   rb    layer_added_compress_methods_tmpr8   module_name_listsrelated_module_name_listscompression_techniquestored_mask	need_maskimnlr@   rm   rmnljr   rh   r   redundancy_clean   s@   


r   c                 C   s  t t|}|t }|t }|t }dd tt|D }|t }	 t|t|ks,J t||D ]/\}	}
t	| |d t
|	 }t	||d t
|
 }t| | D ]\}}|j|j qTq1|D ]%}t	| |}t	||}t| t| | D ]\}}|j|j q|qcdS )aI  
    Given a student model and a teacher model, select the
    Args:
        student_model (`torch.nn.Module`)
            The model we will update weight
        teacher_model (`torch.nn.Module`)
            The model guide the student to learn
        deepspeed_config (`DeepSpeedConfig`)
            The path of ds_config
    c                 S   s   g | ]}|qS r   r   ).0r{   r   r   r   
<listcomp>   s    z*student_initialization.<locals>.<listcomp>r   N)r   r   r-   MODULE_NAME_PREFIXTEACHER_LAYERrangelenOTHER_MODULE_NAMEr2   r   str
parametersdatacopy_print)student_modelra   r`   r   r7   module_name_prefixteacher_layerstudent_layerother_module_names_namet_names_modulet_modules_paramt_paramr'   r   r   r   rR      s,   

rR   )NT)N)NN)r   rX   r   r   r   r   r   r   runtime.config_utilsr	   	constantsr   r   neural_compressorrW   ImportErrorer   r*   rF   rc   r   rR   r   r   r   r   <module>   s&   



2
0,