o
    wi%                     @   s~   d dl Zd dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZ G d	d
 d
ZdS )    N)finetunepretrain)MockDataModule)SquadDataModule)MistralConfig7BMistralModel)LoRA)
mistral_7b)Trainerc                   @   sj   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	ej
dg ddd Zdd Zdd ZdS )TestMistralclass)scopec                 C   s   t S N)mistral)self r   g/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/tests/collections/llm/recipes/test_mistral.pyrecipe_module   s   zTestMistral.recipe_modulec                 C   sL   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J d S r   )model
isinstancerunConfig__fn_or_cls__r   configr   )r   r   model_configr   r   r   
test_model    s
   zTestMistral.test_modelc                 C   s   |  }t|tjsJ |jtksJ |jdksJ |jdks!J |jdks(J t|j	tjs1J |j	jj
dks:J |j	jdksBJ |j	jdksJJ |j	jd u sRJ |j	jd u sZJ |j	jdksbJ |j	jdu sjJ d S )Ngpu      MegatronStrategy   F)trainerr   r   r   r   r
   acceleratordevices	num_nodesstrategy__name__tensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtype$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallelr   r   trainer_configr   r   r   test_trainer'   s   zTestMistral.test_trainerc                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J d S )N   i   r   )pretrain_reciper   r   Partialr   r   r   r   r   r!   r
   datar   
seq_lengthglobal_batch_sizemicro_batch_sizer   r   reciper   r   r   test_pretrain_recipe9   s   z TestMistral.test_pretrain_recipec                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J t|jtjsgJ |jjtksoJ d S )Ni      r   )finetune_reciper   r   r2   r   r   r   r   r   r!   r
   r3   r   r4   r5   r6   peftr   r7   r   r   r   test_finetune_recipeG   s   z TestMistral.test_finetune_recipeznum_nodes,num_gpus_per_node))r   r   )r       )r>   r    c                 C   s2   |j ||d}|jj|ksJ |jj|ksJ d S )N)r$   num_gpus_per_node)r1   r!   r$   r#   )r   r   r$   r?   r8   r   r   r   2test_pretrain_recipe_with_different_configurationsW   s   z>TestMistral.test_pretrain_recipe_with_different_configurationsc                 C   sV   |j ddddd}|jjdksJ |jjdksJ |jjdks!J |jjdu s)J d S )Nr    r>   T)tensor_parallelismpipeline_parallelismcontext_parallelismsequence_parallelism)r!   r%   r'   r(   r+   r,   r-   r   r   r    test_trainer_parallelism_options]   s   z,TestMistral.test_trainer_parallelism_optionsc                 C   sJ   |  }|j}|jdksJ |jdksJ |jdksJ |jdks#J d S )N    r0   i   )r   r   
num_layershidden_sizenum_attention_headsr4   )r   r   r   mistral_configr   r   r   test_model_config_parametersf   s   z(TestMistral.test_model_config_parametersN)r&   
__module____qualname__pytestfixturer   r   r/   r9   r=   markparametrizer@   rE   rK   r   r   r   r   r      s    


	r   )nemo_runr   rN   nemo.collections.llm.apir   r   "nemo.collections.llm.gpt.data.mockr   #nemo.collections.llm.gpt.data.squadr   &nemo.collections.llm.gpt.model.mistralr   r   nemo.collections.llm.peft.lorar   nemo.collections.llm.recipesr	   r   nemo.lightningr
   r   r   r   r   r   <module>   s   