o
    }oi                     @   sb   d dl Zd dlZd dlZd dlmZ d dlmZmZm	Z	 d dl
mZ d dlmZ G dd dZdS )    N)finetune)LoRAMLlamaConfig90BInstructMLlamaModel
mllama_90b)Trainerc                   @   sZ   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	ej
dddgdd ZdS )TestMLLama90Bclass)scopec                 C   s   t S Nr   )self r   a/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/vlm/recipes/test_mllama_90b.pyrecipe_module   s   zTestMLLama90B.recipe_modulec                 C   sL   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J d S r   )model
isinstancerunConfig__fn_or_cls__r   configr   )r   r   model_configr   r   r   
test_model   s
   zTestMLLama90B.test_modelc                 C   s  |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J |j	jdks=J |j	jdksEJ |j	jdksMJ |j	j}t|tjsYJ |jdks`J |jdksgJ |jdksnJ |jtjksvJ |j}t|tjsJ |jjdksJ |jdksJ |jdksJ |jdksJ |jd	ksJ |jd
ksJ |jdksJ |jdksJ |jd usJ |jd usJ d S )Ngpu      r   MockDataModulei  i         i  )0  r   ) finetune_reciper   r   Partialr   r   r   r   r   trainerr   accelerator	num_nodesdevicesstrategytensor_model_parallel_sizepipeline_model_parallel_size$encoder_pipeline_model_parallel_sizepipeline_dtypetorchbfloat16data__name__
seq_lengthdecoder_seq_lengthglobal_batch_sizemicro_batch_size
vocab_size	crop_sizenum_workerslogresume)r   r   recipestratr-   r   r   r   test_finetune_recipe_default(   s8   z*TestMLLama90B.test_finetune_recipe_defaultc                 C   s   |j dd}t|dsJ t|jtjsJ |jjtksJ |j}|jdu s(J g d}|j	|ks3J |j
dks:J |jdksAJ |jdksHJ |jd	ksOJ |jjjd
ksXJ d S )Nlorapeft_schemepeftT)
linear_qkvlinear_q	linear_kvr       g?preg-C6?)r    hasattrr   r>   r   r   r   r   freeze_vision_modeltarget_modulesdimalphadropoutdropout_positionoptimr   lr)r   r   r8   peft_configexpected_modulesr   r   r   test_finetune_recipe_peft_loraS   s   z,TestMLLama90B.test_finetune_recipe_peft_lorac                 C   s<   t t |jdd W d    d S 1 sw   Y  d S )Nnoner<   )pytestraises
ValueErrorr    )r   r   r   r   r   test_invalid_peft_schemeh   s   "z&TestMLLama90B.test_invalid_peft_schemeznum_nodes, num_gpus)r   r   )r      c                 C   s2   |j ||d}|jj|ksJ |jj|ksJ d S )N)r$   num_gpus_per_node)r    r"   r$   r%   )r   r   r$   num_gpusr8   r   r   r   -test_finetune_recipe_different_configurationsm   s   z;TestMLLama90B.test_finetune_recipe_different_configurationsN)r.   
__module____qualname__rQ   fixturer   r   r:   rO   rT   markparametrizerX   r   r   r   r   r	      s    

	+r	   )nemo_runr   rQ   r+   nemo.collections.llm.apir   nemo.collections.vlmr   r   r   nemo.collections.vlm.recipesr   nemo.lightningr   r	   r   r   r   r   <module>   s   