o
    }oig                     @   s   d dl Zd dlZd dlmZ d dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZ G d
d dZdS )    N)pretrain)MockDataModule)GPTConfig175BGPTModel)	gpt3_175b)Trainer)GarbageCollectionCallback)MegatronCommOverlapCallback)TimingCallbackc                   @   sL   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dS )TestGPT3_175Bc                 C   sL   t  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J d S N)	r   model
isinstancerunConfig__fn_or_cls__r   configr   )selfmodel_config r   `/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_gpt3_175b.py
test_model   s
   zTestGPT3_175B.test_modelc                 C   sV  t  }t|tjsJ |jtksJ |jjdksJ |jj	dks#J |jj
tjks,J |jjdks4J |jjdks<J |jjdu sDJ |jdksKJ |jdksRJ |jdksYJ |jdks`J |jd	ksgJ |jd
ksnJ |jdksuJ |jdks|J |jjjdu sJ |jjjdu sJ |jjjdu sJ |jjjdu sJ |jjjdu sJ d S )N            Ti{ 2       
   i  @   )r   trainerr   r   r   r   r   strategytensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtypetorchbfloat16$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallel	max_stepsaccumulate_grad_batcheslimit_test_batcheslimit_val_batcheslog_every_n_stepsval_check_interval	num_nodesdevicesddpcheck_for_nan_in_gradgrad_reduce_in_fp32overlap_grad_reduceoverlap_param_gatheraverage_in_collectiver   r    r   r   r   test_trainer_default_settings$   s,   z+TestGPT3_175B.test_trainer_default_settingsc              
   C   s   t jddddddddd}|jjdksJ |jjdksJ |jjdks%J |jjdks-J |jjdu s5J |jdks<J |j	dksCJ |j
dksJJ d S )Nr   r      Fr   i  )tensor_parallelismpipeline_parallelismvirtual_pipeline_parallelismcontext_parallelismsequence_parallelismr0   num_gpus_per_noder*   )r   r    r!   r"   r#   r'   r(   r)   r*   r0   r1   r8   r   r   r   test_trainer_custom_settingsB   s$   z*TestGPT3_175B.test_trainer_custom_settingsc                 C   s*   t tg}tj|d}|j|ksJ d S )N)	callbacks)r   r   r
   r   r    rB   )r   rB   r    r   r   r   test_trainer_with_callbacksZ   s   z)TestGPT3_175B.test_trainer_with_callbacksc                 C   s   t  }t|tjsJ |jtksJ t|jtjsJ |jjt	ks$J t|jj
tjs.J |jj
jtks7J t|jtjs@J |jjtksHJ |jjdksPJ |jjdksXJ |jjdks`J d S )Ni   r:   )r   pretrain_reciper   r   Partialr   r   r   r   r   r   r   datar   
seq_lengthglobal_batch_sizemicro_batch_sizer   reciper   r   r   %test_pretrain_recipe_default_settings_   s   z3TestGPT3_175B.test_pretrain_recipe_default_settingsc                 C   s4   t jdddd}|jjdksJ |jjdksJ d S )N
custom_runr   r   )namer0   r@   )r   rD   r    r0   r1   rJ   r   r   r   $test_pretrain_recipe_custom_settingsq   s   z2TestGPT3_175B.test_pretrain_recipe_custom_settingsc                 C   s   t  }t |}tdd |jjD sJ tdd |jjD s#J tdd |jjD }|jdu s5J |jdu s<J |j	dksCJ |j
du sJJ |jjjdu sSJ d S )Nc                 s   &    | ]}t |tjo|jtkV  qd S r   r   r   r   r   r   .0cbr   r   r   	<genexpr>   
    
zHTestGPT3_175B.test_pretrain_performance_optimizations.<locals>.<genexpr>c                 s   rP   r   r   r   r   r   r	   rR   r   r   r   rU      rV   c                 s   s*    | ]}t |tjr|jtkr|V  qd S r   rW   rR   r   r   r   rU      s    

Tr   F)r   rD   "pretrain_performance_optimizationsanyr    rB   nexttp_comm_overlapdefer_embedding_wgrad_computewgrad_deferral_limit(overlap_param_gather_with_optimizer_steppluginsr4   )r   base_reciperK   comm_overlap_cbr   r   r   'test_pretrain_performance_optimizationsz   s    
z5TestGPT3_175B.test_pretrain_performance_optimizationsc                 C   sV   t jdd}tdd |jjD sJ tdd |jjD s J |jjjdu s)J d S )NT)performance_modec                 s   rP   r   rQ   rR   r   r   r   rU      rV   zKTestGPT3_175B.test_pretrain_recipe_with_performance_mode.<locals>.<genexpr>c                 s   rP   r   rW   rR   r   r   r   rU      rV   F)r   rD   rY   r    rB   r_   r4   rJ   r   r   r   *test_pretrain_recipe_with_performance_mode   s   z8TestGPT3_175B.test_pretrain_recipe_with_performance_modeN)__name__
__module____qualname__r   r9   rA   rC   rL   rO   rb   rd   r   r   r   r   r      s    	r   )nemo_runr   r%   nemo.collections.llm.apir   "nemo.collections.llm.gpt.data.mockr   nemo.collections.llm.gpt.modelr   r   nemo.collections.llm.recipesr   nemo.lightningr   3nemo.lightning.pytorch.callbacks.garbage_collectionr   6nemo.lightning.pytorch.callbacks.megatron_comm_overlapr	   nemo.utils.exp_managerr
   r   r   r   r   r   <module>   s   