o
    }oim                     @   s~   d dl Zd dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZ G d	d
 d
ZdS )    N)finetunepretrain)MockDataModule)SquadDataModule)Llama32Config3B
LlamaModel)LoRA
llama32_3b)Trainerc                   @   sX   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	dd Z
dd Zdd ZdS )TestLlama32_3Bclass)scopec                 C   s   t S )Nr	   )self r   a/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_llama32_3b.pyrecipe_module   s   zTestLlama32_3B.recipe_modulec                 C   s\   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J |jjdks,J d S )N    )	model
isinstancerunConfig__fn_or_cls__r   configr   
seq_length)r   r   model_configr   r   r   
test_model    s   zTestLlama32_3B.test_modelc                 C   s   |  }t|tjsJ |jtksJ |jdksJ |jdks!J |jdks(J |j	dks/J t|j
tjs8J |j
jjdksAJ |j
jdksIJ |j
jdksQJ |j
jd u sYJ |j
jd u saJ |j
jdksiJ |j
jdu sqJ d S )Ngpu      i{ MegatronStrategyF)trainerr   r   r   r   r   acceleratordevices	num_nodes	max_stepsstrategy__name__tensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtype$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallel)r   r   trainer_configr   r   r   test_trainer(   s   zTestLlama32_3B.test_trainerc                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J d S )Nr   i   r   )pretrain_reciper   r   Partialr   r   r   r   r   r!   r   datar   r   global_batch_sizemicro_batch_sizer   r   reciper   r   r   test_pretrain_recipe;   s   z#TestLlama32_3B.test_pretrain_recipec                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ t|jtjsWJ |jjtks_J |jjdksgJ |jjdksoJ |jjjdksxJ d S )Ni   r      g-C6?)finetune_reciper   r   r1   r   r   r   r   r   r!   r   r2   r   r   peftr   dimalphaoptimr   lrr5   r   r   r   test_finetune_recipeI   s   z#TestLlama32_3B.test_finetune_recipec                 C   sV   |j dd}|jjdksJ |jjddiksJ t|jds J |jjjdks)J d S )NT)packed_sequencei   pad_to_max_lengthpacked_sequence_specs)r9   r2   r   dataset_kwargshasattrrB   packed_sequence_sizer5   r   r   r   )test_finetune_recipe_with_packed_sequenceZ   s
   z8TestLlama32_3B.test_finetune_recipe_with_packed_sequencec                 C   sP   |j d d}|jjjdksJ |jjjdksJ t|dr$|jd u s&J d S d S )Npeft_schemer   gh㈵>r:   )	r9   r!   r&   r(   r=   r   r>   rD   r:   r5   r   r   r   !test_finetune_recipe_without_pefta   s    z0TestLlama32_3B.test_finetune_recipe_without_peftc                 C   s@   t jtdd |jdd W d    d S 1 sw   Y  d S )Nz(Unrecognized peft scheme: invalid_scheme)matchinvalid_schemerG   )pytestraises
ValueErrorr9   )r   r   r   r   r   &test_finetune_recipe_with_invalid_peftg   s   "z5TestLlama32_3B.test_finetune_recipe_with_invalid_peftN)r'   
__module____qualname__rL   fixturer   r   r/   r7   r?   rF   rI   rO   r   r   r   r   r      s    

r   )nemo_runr   rL   nemo.collections.llm.apir   r   "nemo.collections.llm.gpt.data.mockr   #nemo.collections.llm.gpt.data.squadr   $nemo.collections.llm.gpt.model.llamar   r   nemo.collections.llm.peft.lorar   nemo.collections.llm.recipesr
   nemo.lightningr   r   r   r   r   r   <module>   s   