o
    }ois                     @   s~   d dl Zd dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZmZ d dlmZ G d	d
 d
ZdS )    N)finetunepretrain)LoRAt5_220m)MockDataModule)SquadDataModule)T5Config220MT5Model)Trainerc                   @   sj   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	ej
dg ddd Zdd Zdd ZdS )TestT5_220Mclass)scopec                 C   s   t S Nr   )self r   ^/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_t5_220m.pyrecipe_module   s   zTestT5_220M.recipe_modulec                 C   sL   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J d S r   )model
isinstancerunConfig__fn_or_cls__r
   configr	   )r   r   model_configr   r   r   
test_model    s
   zTestT5_220M.test_modelc                 C   s  |  }t|tjsJ |jtksJ |jdksJ |jdks!J |jdks(J |j	dks/J t|j
tjs8J |j
jjdksAJ |j
jdksIJ |j
jdksQJ |j
jd u sYJ |j
jd u saJ |j
jdksiJ |j
jdu sqJ |j
jdu syJ |j
jdu sJ |j
jdu sJ |jdksJ |jdksJ |jd	ksJ |jd
ksJ |jdu sJ |jdksJ t|jtjsJ |jjjdksJ d S )Ngpu      i@B MegatronStrategyFT2       
   i  MegatronMixedPrecision)trainerr   r   r   r   r   acceleratordevices	num_nodes	max_stepsstrategy__name__tensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtype$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallelgradient_as_bucket_viewckpt_async_saveckpt_parallel_loadaccumulate_grad_batcheslimit_test_batcheslimit_val_batcheslog_every_n_stepsuse_distributed_samplerval_check_intervalpluginsr   r   trainer_configr   r   r   test_trainer'   s4   zTestT5_220M.test_trainerc                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J d S Ni      )pretrain_reciper   r   Partialr   r   r   r   r
   r$   r   datar   
seq_lengthseq_length_decglobal_batch_sizer   r   reciper   r   r   test_pretrain_recipeI   s   z TestT5_220M.test_pretrain_recipec                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J t|jtjsgJ |jjtksoJ d S r>   )finetune_reciper   r   rA   r   r   r   r   r
   r$   r   rB   r   rC   rD   rE   peftr   rF   r   r   r   test_finetune_recipeW   s   z TestT5_220M.test_finetune_recipeznum_nodes,num_gpus_per_node))r   r   )      )rM   rL   c                 C   s2   |j ||d}|jj|ksJ |jj|ksJ d S )N)r'   num_gpus_per_node)r@   r$   r'   r&   )r   r   r'   rN   rG   r   r   r   2test_pretrain_recipe_with_different_configurationsg   s   z>TestT5_220M.test_pretrain_recipe_with_different_configurationsc                 C   s2   |j ddd}|jjdksJ |jjdksJ d S )NrL   )tensor_parallelismpipeline_parallelism)r$   r)   r+   r,   r;   r   r   r    test_trainer_parallelism_optionsm   s   z,TestT5_220M.test_trainer_parallelism_optionsc                 C   sJ   |  }|j}|jdksJ |jdksJ |jdksJ |jdks#J d S )N   i   i   )r   r   
num_layershidden_sizeffn_hidden_sizenum_attention_heads)r   r   r   llama_configr   r   r   test_model_config_parametersu   s   z(TestT5_220M.test_model_config_parametersN)r*   
__module____qualname__pytestfixturer   r   r=   rH   rK   markparametrizerO   rR   rY   r   r   r   r   r      s    

"
r   )nemo_runr   r\   nemo.collections.llm.apir   r   nemo.collections.llm.peft.lorar   nemo.collections.llm.recipesr   !nemo.collections.llm.t5.data.mockr   "nemo.collections.llm.t5.data.squadr    nemo.collections.llm.t5.model.t5r	   r
   nemo.lightningr   r   r   r   r   r   <module>   s   