o
    }oi                     @   s~   d dl Zd dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZmZ d dlmZ G d	d
 d
ZdS )    N)finetunepretrain)LoRAt5_11b)MockDataModule)SquadDataModule)T5Config11BT5Model)Trainerc                   @   sj   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	ej
dg ddd Zdd Zdd ZdS )
TestT5_11Bclass)scopec                 C   s   t S Nr   )self r   ]/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_t5_11b.pyrecipe_module   s   zTestT5_11B.recipe_modulec                 C   sL   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J d S r   )model
isinstancerunConfig__fn_or_cls__r
   configr	   )r   r   model_configr   r   r   
test_model    s
   zTestT5_11B.test_modelc                 C   s  |  }t|tjsJ |jtksJ |jdksJ |jdks!J |jdks(J |j	dks/J t|j
tjs8J |j
jjdksAJ |j
jdksIJ |j
jdksQJ |j
jd u sYJ |j
jd u saJ |j
jdksiJ |j
jdu sqJ |j
jd	u syJ |j
jd	u sJ |j
jd	u sJ |jdksJ |jd
ksJ |jdksJ |jdksJ |jdu sJ |jdksJ t|jtjsJ |jjjdksJ d S )Ngpu      i@B MegatronStrategy      FT2       
   i  MegatronMixedPrecision)trainerr   r   r   r   r   acceleratordevices	num_nodes	max_stepsstrategy__name__tensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtype$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallelgradient_as_bucket_viewckpt_async_saveckpt_parallel_loadaccumulate_grad_batcheslimit_test_batcheslimit_val_batcheslog_every_n_stepsuse_distributed_samplerval_check_intervalpluginsr   r   trainer_configr   r   r   test_trainer'   s4   zTestT5_11B.test_trainerc                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J d S )N      i  )pretrain_reciper   r   Partialr   r   r   r   r
   r&   r   datar   
seq_lengthseq_length_decglobal_batch_sizer   r   reciper   r   r   test_pretrain_recipeI   s   zTestT5_11B.test_pretrain_recipec                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J t|jtjsgJ |jjtksoJ d S )Nr@   rA   )finetune_reciper   r   rC   r   r   r   r   r
   r&   r   rD   r   rE   rF   rG   peftr   rH   r   r   r   test_finetune_recipeW   s   zTestT5_11B.test_finetune_recipeznum_nodes,num_gpus_per_node))r!   r   )   r    )r    rN   c                 C   s2   |j ||d}|jj|ksJ |jj|ksJ d S )N)r)   num_gpus_per_node)rB   r&   r)   r(   )r   r   r)   rO   rI   r   r   r   2test_pretrain_recipe_with_different_configurationsg   s   z=TestT5_11B.test_pretrain_recipe_with_different_configurationsc                 C   s2   |j ddd}|jjdksJ |jjdksJ d S )NrN   )tensor_parallelismpipeline_parallelism)r&   r+   r-   r.   r=   r   r   r    test_trainer_parallelism_optionsm   s   z+TestT5_11B.test_trainer_parallelism_optionsc                 C   sX   |  }|j}|jdksJ |jdksJ |jdksJ |jdks#J |jdks*J d S )N   i   i (  @   )r   r   
num_layersencoder_num_layershidden_sizeffn_hidden_sizenum_attention_heads)r   r   r   llama_configr   r   r   test_model_config_parametersu   s   z'TestT5_11B.test_model_config_parametersN)r,   
__module____qualname__pytestfixturer   r   r?   rJ   rM   markparametrizerP   rS   r\   r   r   r   r   r      s    

"
r   )nemo_runr   r_   nemo.collections.llm.apir   r   nemo.collections.llm.peft.lorar   nemo.collections.llm.recipesr   !nemo.collections.llm.t5.data.mockr   "nemo.collections.llm.t5.data.squadr    nemo.collections.llm.t5.model.t5r	   r
   nemo.lightningr   r   r   r   r   r   <module>   s   