o
    TÃië	  ã                   @   sB   d dl Z ddlmZmZ d dlmZ d dlmZ G dd„ dƒZ	dS )é    Né   )ÚTPTrainingConfigÚTPConfig)Úgroupsc                   @   s>   e Zd Zdd„ Zdd„ Zdd„ Zddd	„Zd
d„ Zdd„ ZdS )ÚTpTrainingManagerc                 C   sš   || _ |  |¡| _ddlm} ddlm} | |¡}td|ƒ |  	|¡ |  
¡  |ƒ  ¡  |ƒ  ¡ ¡}t |t ¡ | jj¡ |ƒ  | ¡ ¡ |  |¡ d S )Nr   )ÚAutoTP)Úget_acceleratorzAutoTP: )ÚmoduleÚ_initialize_configÚconfigÚdeepspeed.module_inject.auto_tpr   Ú	deepspeedr   Ú	tp_parserÚprintÚ_initialize_tp_configÚ_get_model_config_generateÚget_rng_stateÚtoÚcurrent_device_nameÚdistÚ	broadcastr   Ú"get_tensor_model_parallel_src_rankÚ	tp_configÚtp_groupÚset_rng_stateÚcpuÚ_apply_policies)ÚselfÚmodelÚtp_sizeÚdtyper   r   Úparser_dictÚ
_rng_state© r#   ú`/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/tensor_parallel/tp_manager.pyÚ__init__   s   


zTpTrainingManager.__init__c                 C   s   t ƒ }||_|S )z>Initialize and return the DeepSpeed TP training configuration.)r   r    )r   r    r   r#   r#   r$   r
   %   s   z$TpTrainingManager._initialize_configc                 C   s(   |D ]\}}|| j _|  | j |¡ qdS )z/Apply injection policies to the parsed modules.N)r   Úinjection_policy_tupleÚ_apply_injection_policy)r   r!   Úclient_moduleÚinjection_policyr#   r#   r$   r   +   s   þz!TpTrainingManager._apply_policiesNc                 C   s<   ddl m} 	 t| jtjjƒr||| jd | j| jƒ d S d S )Nr   )Úreplace_transformer_layer)	Údeepspeed.module_injectr*   Ú
isinstancer	   ÚtorchÚnnÚModuler   Úmodel_config)r   r   r(   r*   r#   r#   r$   r'   1   s
   ÿz)TpTrainingManager._apply_injection_policyc                 C   s4   t ƒ | _|| j_t |¡ t ¡ | j_| j| j_dS )z(Perform TP configuration initialization.N)	r   r   r   r   Ú_init_tp_mesh_deviceÚget_tensor_model_parallel_groupr   r   Útensor_parallel)r   r   r#   r#   r$   r   7   s
   
z'TpTrainingManager._initialize_tp_configc                 C   s   t | jddƒ| _dS )z+Generate and apply HF model  configuration.r   N)Úgetattrr	   r0   )r   r#   r#   r$   r   @   s   z,TpTrainingManager._get_model_config_generate)N)	Ú__name__Ú
__module__Ú__qualname__r%   r
   r   r'   r   r   r#   r#   r#   r$   r      s    
	r   )
r-   r   r   r   Údeepspeed.utilsr   Údeepspeed.commÚcommr   r   r#   r#   r#   r$   Ú<module>   s
   