o
    }oi                     @   sf   d dl Zd dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ G dd dZdS )	    N)pretrain)BERTMockDataModule	bert_110m)Trainer)TimingCallbackc                   @   sZ   e Zd Zejdddd Zdd Zdd Zd	d
 Zej	
dg ddd Zdd ZdS )TestBERT_110Mclass)scopec                 C   s   t S Nr   )self r   `/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_bert_110m.pyrecipe_module   s   zTestBERT_110M.recipe_modulec                 C   s   |  }t|tjsJ d S r   )model
isinstancerunConfig)r   r   model_configr   r   r   
test_model   s   zTestBERT_110M.test_modelc                 C   sH  |  j}t|tjsJ |jtksJ |jdksJ |jdks"J |j	dks)J |j
dks0J t|jtjs9J |jjjdksBJ |jjdksJJ |jjdksRJ |jjtjks[J |jjd u scJ |jjdkskJ |jjdu ssJ |jdkszJ |jdksJ |jdksJ |jdksJ |jd	ksJ td
d |jD sJ d S )Ngpu      i{ MegatronStrategyF    
   i  c                 s   s&    | ]}t |tjo|jtkV  qd S r   )r   r   r   __fn_or_cls__r   ).0cbr   r   r   	<genexpr>@   s    
z-TestBERT_110M.test_trainer.<locals>.<genexpr>)pretrain_recipetrainerr   r   r   r   r   acceleratordevices	num_nodes	max_stepsstrategy__name__tensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtypetorchbfloat16$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallelaccumulate_grad_batcheslimit_test_batcheslimit_val_batcheslog_every_n_stepsval_check_intervalany	callbacksr   r   trainer_configr   r   r   test_trainer%   s.   
zTestBERT_110M.test_trainerc                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J |j	jdks=J |j	jdksEJ |j	jdksMJ d S )Ni   r      )r    r   r   Partialr   r   r!   r   r   datar   
seq_lengthglobal_batch_sizemicro_batch_size)r   r   reciper   r   r   test_pretrain_recipeD   s   z"TestBERT_110M.test_pretrain_recipeznum_nodes,num_gpus_per_node))r   r   )r:      )rB   r:   c                 C   s2   |j ||d}|jj|ksJ |jj|ksJ d S )N)r$   num_gpus_per_node)r    r!   r$   r#   )r   r   r$   rC   r@   r   r   r   2test_pretrain_recipe_with_different_configurationsP   s   z@TestBERT_110M.test_pretrain_recipe_with_different_configurationsc                 C   s   |j ddtjddddj}|jjdksJ |jjdksJ |jjtjks&J |jjdks.J |jj	dks6J |jj
du s>J d S )Nr:   rB   T)tensor_parallelismpipeline_parallelismpipeline_parallelism_typevirtual_pipeline_parallelismcontext_parallelismsequence_parallelism)r    r+   r,   r!   r&   r(   r)   r*   r-   r.   r/   r7   r   r   r    test_trainer_parallelism_optionsV   s    	z.TestBERT_110M.test_trainer_parallelism_optionsN)r'   
__module____qualname__pytestfixturer   r   r9   rA   markparametrizerD   rK   r   r   r   r   r      s    


r   )nemo_runr   rN   r+   nemo.collections.llm.apir   #nemo.collections.llm.bert.data.mockr   nemo.collections.llm.recipesr   nemo.lightningr   nemo.utils.exp_managerr   r   r   r   r   r   <module>   s   