o
    }oi                     @   sV   d dl Zd dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZ G dd dZdS )    N)finetunepretrain)MockDataModule
hyena_base)Trainerc                   @   s   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	ej
dddgdd Zdd Zdd Zdd Zdd Zdd ZdS )TestHyenaBaseclass)scopec                 C   s   t S )Nr   )self r   a/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_hyena_base.pyrecipe_module   s   zTestHyenaBase.recipe_modulec                 C   sV   |j ddd}t|tjsJ |jjdksJ t|jtjs J |jjjdks)J d S )NF   )tp_comm_overlap
seq_length
HyenaModelHyenaTestConfig)model
isinstancerunConfig__fn_or_cls____name__config)r   r   model_configr   r   r   
test_model    s
   zTestHyenaBase.test_modelc                 C   s   |j dddddd}t|tjsJ |jtksJ |jdks J |jdks'J |jdks.J t|j	tjs7J |j	jj
dks@J |j	jdksHJ |j	jdksPJ d S )N            )tensor_parallelismpipeline_parallelism	num_nodesnum_gpus_per_node	max_stepsMegatronStrategy)trainer_reciper   r   r   r   r   devicesr#   r%   strategyr   tensor_model_parallel_sizepipeline_model_parallel_size)r   r   trainer_configr   r   r   test_trainer'   s    zTestHyenaBase.test_trainerc              	   C   s   |j d ddddddd}t|tjsJ |jtksJ t|jtjs$J |jjjdks-J t|j	tjs6J |j	jt
ks>J t|jtjsGJ |jjtksOJ |jjdksWJ |jjdks_J d S )	N   r   r   r   r   test)dataset_configglobal_batch_sizemicro_batch_sizer#   r$   r   
model_sizer   )pretrain_reciper   r   Partialr   r   r   r   r   trainerr   datar   r   r1   r   r   reciper   r   r   test_pretrain_recipe;   s&   	z"TestHyenaBase.test_pretrain_recipec              
   C   s   |j dd ddddddd}t|tjsJ |jtksJ t|jtjs%J |jjjd	ks.J t|j	tjs7J |j	jt
ks?J t|jtjsHJ |jjjd
ksQJ |jjdksYJ |jjdksaJ d S )N
dummy_pathr.   r   r   r   r   r/   )resume_pathr0   r1   r2   r#   r$   r   r3   r   r   )finetune_reciper   r   r5   r   r   r   r   r   r6   r   r7   r   r1   r8   r   r   r   test_finetune_recipeP   s(   
z"TestHyenaBase.test_finetune_recipeznum_nodes,num_gpus_per_node)r   r   )r   r   c                 C   s:   |j ||ddddd}|jj|ksJ |jj|ksJ d S )Nr.   r   r   r/   )r#   r$   r1   r2   r   r3   )r4   r6   r#   r(   )r   r   r#   r$   r9   r   r   r   2test_pretrain_recipe_with_different_configurationsf   s   z@TestHyenaBase.test_pretrain_recipe_with_different_configurationsc                 C   s:   |  }t|tjsJ |jjdksJ |jdksJ d S )Nget_nmt_tokenizerz
byte-level)	tokenizerr   r   r   r   r   library)r   r   tokenizer_configr   r   r   test_tokenizer_recipes   s   z#TestHyenaBase.test_tokenizer_recipec                 C   sD   t jtdd |jdddd W d    d S 1 sw   Y  d S )Nz$Unsupported model size: invalid_size)matchinvalid_sizeFr   )r3   r   r   )pytestraisesNotImplementedErrormodel_recipe)r   r   r   r   r   test_invalid_model_sizey   s   "z%TestHyenaBase.test_invalid_model_sizec              
   C   s>   |j d dddddddd}|jd usJ |jjjdksJ d S )	Nr.   r   r   r   r   r/   test_project)r0   r1   r2   r#   r$   r   r3   wandb_project)r4   logwandbprojectr8   r   r   r   test_wandb_logger}   s   zTestHyenaBase.test_wandb_loggerc              
   C   s8   |j d dddddddd}td	d
 |jjD sJ d S )Nr.   r   r   r   r   r/   T)r0   r1   r2   r#   r$   r   r3   tflops_callbackc                 s       | ]	}|j jd kV  qdS )FLOPsMeasurementCallbackNr   r   .0callbackr   r   r   	<genexpr>   s    
z/TestHyenaBase.test_callbacks.<locals>.<genexpr>r4   anyr6   	callbacksr8   r   r   r   test_callbacks   s   
zTestHyenaBase.test_callbacksc                 C   s<   |j d ddddddddd	d

}tdd |jjD sJ d S )Nr.   r   r   r   r   r/   T
      )
r0   r1   r2   r#   r$   r   r3   nsys_profilingnsys_start_stepnsys_end_stepc                 s   rS   )NsysCallbackNrU   rV   r   r   r   rY      s    z4TestHyenaBase.test_nsys_profiling.<locals>.<genexpr>rZ   r8   r   r   r   test_nsys_profiling   s   z!TestHyenaBase.test_nsys_profilingN)r   
__module____qualname__rG   fixturer   r   r-   r:   r>   markparametrizer?   rD   rK   rQ   r]   rd   r   r   r   r   r      s    


r   )nemo_runr   rG   nemo.collections.llm.apir   r   "nemo.collections.llm.gpt.data.mockr   nemo.collections.llm.recipesr   nemo.lightningr   r   r   r   r   r   <module>   s   