o
    }oim                     @   s~   d dl Zd dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZ G d	d
 d
ZdS )    N)finetunepretrain)MockDataModule)SquadDataModule)DeepSeekModelDeepSeekV2Config)LoRAdeepseek_v2)Trainerc                   @   sH   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	dd Z
dS )TestDeepSeekV2class)scopec                 C   s   t S Nr	   )self r   b/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_deepseek_v2.pyrecipe_module   s   zTestDeepSeekV2.recipe_modulec                 C   sL   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J d S r   )model
isinstancerunConfig__fn_or_cls__r   configr   )r   r   model_configr   r   r   
test_model    s
   zTestDeepSeekV2.test_modelc                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J |j	jjdksgJ |j	jjdkspJ d S )Ni   i             )pretrain_reciper   r   Partialr   r   r   r   r   trainerr   datar   
seq_lengthglobal_batch_sizemicro_batch_sizestrategypipeline_model_parallel_sizeexpert_model_parallel_sizer   r   reciper   r   r   test_pretrain_recipe'   s   z#TestDeepSeekV2.test_pretrain_recipec                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ t|jtjsWJ |jjtks_J |jjjdkshJ |jjg dksrJ d S )Ni   g-C6?)linear_q_down_projlinear_q_up_projlinear_kv_down_projlinear_kv_up_projlinear_proj)finetune_reciper   r   r    r   r   r   r   r   r!   r   r"   r   r#   peftr   optimr   lrtarget_modulesr)   r   r   r   test_finetune_recipe9   s   z#TestDeepSeekV2.test_finetune_recipec                 C   sb   |j d d}|jjjdksJ |jjjdksJ |jjjdks!J t|dr-|j	d u s/J d S d S )Npeft_schemer   r   gh㈵>r2   )
r1   r!   r&   r'   r(   r3   r   r4   hasattrr2   r)   r   r   r   !test_finetune_recipe_without_peftQ   s
    z0TestDeepSeekV2.test_finetune_recipe_without_peftc                 C   s@   t jtdd |jdd W d    d S 1 sw   Y  d S )Nz(Unrecognized peft scheme: invalid_scheme)matchinvalid_schemer7   )pytestraises
ValueErrorr1   )r   r   r   r   r   &test_finetune_recipe_with_invalid_peftX   s   "z5TestDeepSeekV2.test_finetune_recipe_with_invalid_peftN)__name__
__module____qualname__r=   fixturer   r   r+   r6   r:   r@   r   r   r   r   r      s    

r   )nemo_runr   r=   nemo.collections.llm.apir   r   "nemo.collections.llm.gpt.data.mockr   #nemo.collections.llm.gpt.data.squadr   'nemo.collections.llm.gpt.model.deepseekr   r   nemo.collections.llm.peft.lorar   nemo.collections.llm.recipesr
   nemo.lightningr   r   r   r   r   r   <module>   s   