o
    }oi                     @   s   d dl Zd dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ G dd dZdS )    N)finetunepretrain)MockDataModule)SquadDataModule)Llama2Config7B
LlamaModel)LoRA	llama2_7b)Trainer)GarbageCollectionCallback)TimingCallbackc                   @   s   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	dd Z
ejdg ddd Zdd Zdd Zdd Zdd Zdd ZdS )TestLlama2_7Bclass)scopec                 C   s   t S Nr	   )self r   `/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_llama2_7b.pyrecipe_module   s   zTestLlama2_7B.recipe_modulec                 C   sL   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J d S r   )model
isinstancerunConfig__fn_or_cls__r   configr   )r   r   model_configr   r   r   
test_model"   s
   zTestLlama2_7B.test_modelc                 C   s   |  }t|tjsJ |jtksJ |jdksJ |jdks!J |jdks(J |j	dks/J t|j
tjs8J |j
jjdksAJ |j
jdksIJ |j
jdksQJ |j
jd u sYJ |j
jd u saJ |j
jdksiJ |j
jdu sqJ d S )Ngpu      i{ MegatronStrategy   F)trainerr   r   r   r   r   acceleratordevices	num_nodes	max_stepsstrategy__name__tensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtype$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallel)r   r   trainer_configr   r   r   test_trainer)   s   zTestLlama2_7B.test_trainerc                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J d S )N   i   r    )pretrain_reciper   r   Partialr   r   r   r   r   r#   r   datar   
seq_lengthglobal_batch_sizemicro_batch_sizer   r   reciper   r   r   test_pretrain_recipe<   s   z"TestLlama2_7B.test_pretrain_recipec                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ t|jtjsWJ |jjtks_J |jjdksgJ |jjdksoJ |jjjdksxJ d S )Ni   r      g-C6?)finetune_reciper   r   r4   r   r   r   r   r   r#   r   r5   r   r6   peftr   dimalphaoptimr   lrr9   r   r   r   test_finetune_recipeJ   s   z"TestLlama2_7B.test_finetune_recipec                 C   sV   |j dd}|jjdksJ |jjddiksJ t|jds J |jjjdks)J d S )NT)packed_sequencer2   pad_to_max_lengthpacked_sequence_specs)r=   r5   r6   dataset_kwargshasattrrF   packed_sequence_sizer9   r   r   r   )test_finetune_recipe_with_packed_sequence[   s
   z7TestLlama2_7B.test_finetune_recipe_with_packed_sequenceznum_nodes,num_gpus_per_node))r    r   )r"      )rK   r"   c                 C   s2   |j ||d}|jj|ksJ |jj|ksJ d S )N)r&   num_gpus_per_node)r3   r#   r&   r%   )r   r   r&   rL   r:   r   r   r   2test_pretrain_recipe_with_different_configurationsb   s   z@TestLlama2_7B.test_pretrain_recipe_with_different_configurationsc                 C   sP   |j d d}|jjjdksJ |jjjdksJ t|dr$|jd u s&J d S d S )Npeft_schemer"   gh㈵>r>   )	r=   r#   r(   r*   rA   r   rB   rH   r>   r9   r   r   r   !test_finetune_recipe_without_pefth   s    z/TestLlama2_7B.test_finetune_recipe_without_peftc                 C   s@   t jtdd |jdd W d    d S 1 sw   Y  d S )Nz(Unrecognized peft scheme: invalid_scheme)matchinvalid_schemerN   )pytestraises
ValueErrorr=   )r   r   r   r   r   &test_finetune_recipe_with_invalid_peftn   s   "z4TestLlama2_7B.test_finetune_recipe_with_invalid_peftc                 C   sV   |j dd}|jjjdksJ tdd |jjD sJ tdd |jjD s)J d S )NT)performance_moder    c                 s   &    | ]}t |tjo|jtkV  qd S r   )r   r   r   r   r   .0cbr   r   r   	<genexpr>u   s    
zHTestLlama2_7B.test_finetune_performance_optimizations.<locals>.<genexpr>c                 s   rX   r   )r   r   r   r   r   rY   r   r   r   r\   x   s
    
)r=   r#   r(   r*   any	callbacksr9   r   r   r   'test_finetune_performance_optimizationsr   s   z5TestLlama2_7B.test_finetune_performance_optimizationsc                 C   s   |j dd d}|jjjdu sJ |jjjjdu sJ |jjjjdu s$J |jjjjdu s.J |jjjjdu s8J t	dd |jj
D sEJ d S )NTrW   rO   Fc                 s   s(    | ]}t |tjo|jjd kV  qdS )MegatronCommOverlapCallbackN)r   r   r   r   r)   rY   r   r   r   r\      s
    
zUTestLlama2_7B.test_finetune_performance_optimizations_without_peft.<locals>.<genexpr>)r=   r#   pluginsgrad_reduce_in_fp32r(   ddpoverlap_grad_reduceoverlap_param_gatheraverage_in_collectiver]   r^   r9   r   r   r   4test_finetune_performance_optimizations_without_peft}   s   zBTestLlama2_7B.test_finetune_performance_optimizations_without_peftc                 C   s$   |j ddd}|jjdgksJ d S )NTlorar`   
linear_qkv)r=   r>   target_modulesr9   r   r   r   1test_finetune_performance_optimizations_with_peft   s   z?TestLlama2_7B.test_finetune_performance_optimizations_with_peftN)r)   
__module____qualname__rS   fixturer   r   r1   r;   rC   rJ   markparametrizerM   rP   rV   r_   rh   rl   r   r   r   r   r      s    


r   )nemo_runr   rS   nemo.collections.llm.apir   r   "nemo.collections.llm.gpt.data.mockr   #nemo.collections.llm.gpt.data.squadr   $nemo.collections.llm.gpt.model.llamar   r   nemo.collections.llm.peft.lorar   nemo.collections.llm.recipesr
   nemo.lightningr   3nemo.lightning.pytorch.callbacks.garbage_collectionr   nemo.utils.exp_managerr   r   r   r   r   r   <module>   s   