o
    }oi                     @   s   d dl Zd dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
 d dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ G d
d dZdS )    N)finetunepretrain)MockDataModule)SquadDataModule)Baichuan2Config7BBaichuan2Model)LoRAbaichuan2_7b)Trainer)TimingCallbackc                   @   sz   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	ej
dg ddd Zdd Zdd Zdd Zdd ZdS )TestBaichuan2_7Bclass)scopec                 C   s   t S Nr	   )self r   c/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_baichuan2_7b.pyrecipe_module   s   zTestBaichuan2_7B.recipe_modulec                 C   sL   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J d S r   )model
isinstancerunConfig__fn_or_cls__r   configr   )r   r   model_configr   r   r   
test_model"   s
   zTestBaichuan2_7B.test_modelc                 C   s  |  }t|tjsJ |jtksJ |jdksJ |jdks!J |jdks(J |j	dks/J t|j
tjs8J |j
jjdksAJ |j
jdksIJ |j
jdksQJ |j
jd u sYJ |j
jd u saJ |j
jdksiJ |j
jdu sqJ |j
jdu syJ |j
jdu sJ |j
jdu sJ |j
jjdu sJ |j
jjdu sJ |j
jjdu sJ |j
jjdu sJ |jdksJ |jd	ksJ |jd
ksJ |jdksJ |jdu sJ |jdksJ t|j tjsJ |j jjdksJ d S )Ngpu      i{ MegatronStrategy   FT2       
   i  MegatronMixedPrecision)!trainerr   r   r   r   r   acceleratordevices	num_nodes	max_stepsstrategy__name__tensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtype$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallelgradient_as_bucket_viewckpt_async_saveckpt_parallel_loadddpcheck_for_nan_in_gradgrad_reduce_in_fp32overlap_grad_reduceoverlap_param_gatheraccumulate_grad_batcheslimit_test_batcheslimit_val_batcheslog_every_n_stepsuse_distributed_samplerval_check_intervalpluginsr   r   trainer_configr   r   r   test_trainer)   s<   zTestBaichuan2_7B.test_trainerc                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J d S )Ni    i   r   )pretrain_reciper   r   Partialr   r   r   r   r   r&   r   datar   
seq_lengthglobal_batch_sizemicro_batch_sizer   r   reciper   r   r   test_pretrain_recipeQ   s   z%TestBaichuan2_7B.test_pretrain_recipec                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ t|jtjs_J |jjtksgJ |jjjdkspJ d S )Ni      g-C6?)finetune_reciper   r   rF   r   r   r   r   r   r&   r   rG   r   rH   rI   peftr   optimr   lrrK   r   r   r   test_finetune_recipe_   s   z%TestBaichuan2_7B.test_finetune_recipeznum_nodes,num_gpus_per_node))r   r   )r!      )rT   r!   c                 C   s2   |j ||d}|jj|ksJ |jj|ksJ d S )N)r)   num_gpus_per_node)rE   r&   r)   r(   )r   r   r)   rU   rL   r   r   r   2test_pretrain_recipe_with_different_configurationso   s   zCTestBaichuan2_7B.test_pretrain_recipe_with_different_configurationsc                 C   s@   |  }tdd |jjD sJ tdd |jjD sJ d S )Nc                 s   s(    | ]}t |tjo|jjd kV  qdS )MegatronCommOverlapCallbackN)r   r   r   r   r,   .0cbr   r   r   	<genexpr>w   s
    
zKTestBaichuan2_7B.test_pretrain_performance_optimizations.<locals>.<genexpr>c                 s   s&    | ]}t |tjo|jtkV  qd S r   )r   r   r   r   r   rX   r   r   r   r[   {   s    
)pretrain_recipe_performanceanyr&   	callbacksrK   r   r   r   'test_pretrain_performance_optimizationsu   s   z8TestBaichuan2_7B.test_pretrain_performance_optimizationsc                 C   s~   |j ddtjdddd}|jjdksJ |jjdksJ |jjtjks%J |jjdks-J |jjdks5J |jj	du s=J d S )Nr!   rT   T)tensor_parallelismpipeline_parallelismpipeline_parallelism_typevirtual_pipeline_parallelismcontext_parallelismsequence_parallelism)
r&   torchbfloat16r+   r-   r.   r/   r0   r1   r2   rB   r   r   r    test_trainer_parallelism_options   s   z1TestBaichuan2_7B.test_trainer_parallelism_optionsc                 C   sP   |j d d}|jjjdksJ |jjjdksJ t|dr$|jd u s&J d S d S )Npeft_schemer!   gh㈵>rP   )	rO   r&   r+   r-   rQ   r   rR   hasattrrP   rK   r   r   r   !test_finetune_recipe_without_peft   s    z2TestBaichuan2_7B.test_finetune_recipe_without_peftc                 C   s@   t jtdd |jdd W d    d S 1 sw   Y  d S )Nz(Unrecognized peft scheme: invalid_scheme)matchinvalid_schemeri   )pytestraises
ValueErrorrO   )r   r   r   r   r   &test_finetune_recipe_with_invalid_peft   s   "z7TestBaichuan2_7B.test_finetune_recipe_with_invalid_peftN)r,   
__module____qualname__ro   fixturer   r   rD   rM   rS   markparametrizerV   r_   rh   rl   rr   r   r   r   r   r      s    

(

r   )nemo_runr   ro   rf   nemo.collections.llm.apir   r   "nemo.collections.llm.gpt.data.mockr   #nemo.collections.llm.gpt.data.squadr   'nemo.collections.llm.gpt.model.baichuanr   r   nemo.collections.llm.peft.lorar   nemo.collections.llm.recipesr
   nemo.lightningr   nemo.utils.exp_managerr   r   r   r   r   r   <module>   s   