o
    }oi1                     @   s   d dl mZmZmZ d dlZd dlmZ d dlmZmZ G dd dej	Z	G dd dej	Z
dd
dZdddZedddeddd Zedddeddd ZdddZdS )    )ANY	MagicMockpatchN)nn)MegatronStrategy_strategy_libc                       s   e Zd Z fddZ  ZS )Identityc                    s   t    d S N)super__init__self	__class__ U/home/ubuntu/.local/lib/python3.10/site-packages/tests/lightning/test_strategy_lib.pyr      s   zIdentity.__init__)__name__
__module____qualname__r   __classcell__r   r   r   r   r      s    r   c                   @   s   e Zd Zdd ZdS )WithCopyc                 C   s   t  S r	   )r   r   r   r   r   copy   s   zWithCopy.copyN)r   r   r   r   r   r   r   r   r      s    r   returnc                     s   t dddtjd} ddlm  G  fddd}| }|jjdks#J |jjdks+J |jjtjks4J t	
|| j |jjdksCJ |jjdksKJ |jjdksSJ |jjtjks\J d S )N   F)pipeline_model_parallel_sizeexpert_model_parallel_sizesequence_parallelpipeline_dtyper   TransformerConfigc                       s    e Zd Z fddZdd ZdS )z6test_set_model_parallel_attributes.<locals>.DummyModelc                    s    dddddd| _ d S )N   r   F)hidden_sizenum_attention_heads
num_layersnum_moe_expertsadd_bias_linear)configr   r   r   r   r   +   s   
z?test_set_model_parallel_attributes.<locals>.DummyModel.__init__c                 S   s   d S r	   r   r   r   r   r   configure_model0   s   zFtest_set_model_parallel_attributes.<locals>.DummyModel.configure_modelN)r   r   r   r   r'   r   r   r   r   
DummyModel*   s    r(   )r   torchfloat32,megatron.core.transformer.transformer_configr   r&   r   r   r   r   set_model_parallel_attributesparallelismr   )strategyr(   modelr   r   r   "test_set_model_parallel_attributes!   s"   	r0   c            	      C   s2  ddl m}  ddlm} ddlm} | }d|_d|_d|_d|_	d|_
d|_t }d|_d|_d|_d|_d|_	d |_d|_d|_d	|_d |_d	|_tjd
dd|dd	d d
dddddddd ddd	d	d}| D ]&\}}t||s{J d| dt||}||ksJ | d| d| qj|  |   d S )Nr   )#destroy_num_microbatches_calculator)destroy_model_parallelAppStater            F   i  )
world_sizeglobal_rank
local_rankparallel_configseedfp8)r9   r:   r;   tensor_model_parallel_sizer   $virtual_pipeline_model_parallel_sizecontext_parallel_sizer   "pipeline_model_parallel_split_rank$encoder_pipeline_model_parallel_size"encoder_tensor_model_parallel_sizeuse_fp8init_mpi_proc_groupzExpected to find z in AppStatez% in AppState is incorrect, Expected: z	 Actual: ))megatron.core.num_microbatches_calculatorr1   megatron.core.parallel_stater2   
nemo.utilsr4   r?   r   rA   r   r:   r;   r   r@   expert_tensor_parallel_sizerD   rC   tp_comm_overlaprB   use_te_rng_trackerr   init_parallel_ranksitemshasattrgetattr)	r1   r2   r4   	app_statemock_parallel_configexpected_app_statekvapp_attrr   r   r   test_init_parallel_ranks>   sd   	
"
rW   z torch.distributed.is_initializedT)return_valuezmegatron.core.parallel_statec                 G   s   ddl m} | }d|_d|_d|_d |_d |_d|_d|_d|_	d|_
d|_d|_d|_t|  tt  | jjddd d d d d ddddddd dd d S )	Nr   r3   r6   r   Fztp-cp-ep-dp-ppTr?   r   r@   rB   $pipeline_model_parallel_comm_backendrC   rD   rA   r   rJ   	use_sharporder#num_distributed_optimizer_instancesnccl_communicator_config_pathcreate_gloo_process_groups)rI   r4   model_parallel_sizer?   r   rB   rZ   rA   r   rJ   expert_tensor_parallel_rankrF   tensor_model_parallel_rankpipeline_model_parallel_rank	_mpu_tp_2r   init_model_parallelr   r   initialize_model_parallelassert_called_once_withmock_mpuargsr4   rQ   r   r   r   test_init_model_parallelz   sB   
rk   c                 G   s   ddl m} | }d|_d|_d|_d |_d |_d|_d|_d|_	d|_
d|_d|_d|_d|_t|  tt  | jjddd d d d d ddddddd dd d S )	Nr   r3   r6   r   FTztp-cp-ep-pp-dprY   )rI   r4   r`   r?   r   rB   rZ   rA   r   rJ   ra   rF   rb   rc   use_tp_pp_dp_mappingrd   r   re   r   r   rf   rg   rh   r   r   r   &test_init_model_parallel_with_tp_pp_dp   sD   
rm   c                 C   s4   d| j _d| j_d| j_d| j_d| j_d| j_d S )Nr   r   r6   )get_tensor_model_parallel_rankrX    get_pipeline_model_parallel_rank&get_pipeline_model_parallel_world_size!get_pipeline_model_parallel_groupget_tensor_model_parallel_groupget_expert_tensor_parallel_rank)ri   r   r   r   rd   N  s   rd   )r   N)unittest.mockr   r   r   r)   r   nemo.lightningr   r   r   r   r0   rW   rk   rm   rd   r   r   r   r   <module>   s    


<
' *