o
    }oit                     @   s   d dl Zd dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
 d dlmZmZ d dlmZ d dlmZ d dlmZ G d	d
 d
ZdS )    N)finetunepretrain)MockDataModule)SquadDataModule)StarcoderConfig15BStarcoderModel)PEFT_STR2CLSstarcoder_15b)Trainerc                   @   sX   e Zd Zejdddd Zdd Zdd Zd	d
 Zdd Z	dd Z
dd Zdd ZdS )TestStarcoder_15Bclass)scopec                 C   s   t S Nr	   )self r   d/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_starcoder_15b.pyrecipe_module   s   zTestStarcoder_15B.recipe_modulec                 C   sL   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J d S r   )model
isinstancerunConfig__fn_or_cls__r   configr   )r   r   model_configr   r   r   
test_model!   s
   zTestStarcoder_15B.test_modelc                 C   s  |  }t|tjsJ |jtksJ |jdksJ |jdks!J |jdks(J |j	dks/J |j
dks6J |jdks=J |jdksDJ |jdksKJ t|jtjsTJ |jjjdks]J |jjd	kseJ |jjd
ksmJ |jjdu suJ |jjdu s}J |jjdu sJ |jjdu sJ d S )Ngpu      i{ i      
   MegatronStrategy      T)starcoder_trainerr   r   r   r   r   acceleratordevices	num_nodes	max_stepsval_check_intervallimit_test_batcheslimit_val_batcheslog_every_n_stepsstrategy__name__tensor_model_parallel_sizepipeline_model_parallel_sizegradient_as_bucket_viewckpt_include_optimizerckpt_async_saveckpt_parallel_load)r   r   trainer_configr   r   r   test_trainer(   s&   zTestStarcoder_15B.test_trainerc                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ |jjdksNJ |jjdksVJ |jjdks^J |j	jjdksgJ |j	jjdkspJ |j	jjtjkszJ d S )Ni   r   r#   r   r   )pretrain_reciper   r   Partialr   r   r   r   r   trainerr   datar   
seq_lengthglobal_batch_sizemicro_batch_sizer-   r/   r0   pipeline_dtypetorchbfloat16r   r   reciper   r   r   test_pretrain_recipe?   s   z&TestStarcoder_15B.test_pretrain_recipec                 C   s   |  }t|tjsJ |jtksJ t|jtjsJ |jjtks$J t|j	tjs-J |j	jt
ks5J t|jtjs>J |jjtksFJ t|jtjsOJ |jjtd ksYJ |jjjdksbJ d S )Nlora-C6?)finetune_reciper   r   r8   r   r   r   r   r   r9   r   r:   r   peftr   optimr   lrrA   r   r   r   test_finetune_recipeR   s   z&TestStarcoder_15B.test_finetune_recipec                 C   sH   |j dd}t|jtjsJ |jjtd ksJ |jjj	dks"J d S )Ndorapeft_schemerE   )
rF   r   rG   r   r   r   r   rH   r   rI   rA   r   r   r   test_finetune_recipe_with_dorab   s   z0TestStarcoder_15B.test_finetune_recipe_with_dorac                 C   sL   |j d d}t|dr|jd u sJ |jjjdksJ |jjjdks$J d S )NrL   rG   r   gh㈵>)	rF   hasattrrG   r9   r-   r0   rH   r   rI   rA   r   r   r   !test_finetune_recipe_without_pefth   s   z3TestStarcoder_15B.test_finetune_recipe_without_peftc                 C   s@   t jtdd |jdd W d    d S 1 sw   Y  d S )Nz(Unrecognized peft scheme: invalid_scheme)matchinvalid_schemerL   )pytestraises
ValueErrorrF   )r   r   r   r   r   &test_finetune_recipe_with_invalid_peftn   s   "z8TestStarcoder_15B.test_finetune_recipe_with_invalid_peftN)r.   
__module____qualname__rS   fixturer   r   r6   rC   rJ   rN   rP   rV   r   r   r   r   r      s    

r   )nemo_runr   rS   r?   nemo.collections.llm.apir   r   "nemo.collections.llm.gpt.data.mockr   #nemo.collections.llm.gpt.data.squadr   (nemo.collections.llm.gpt.model.starcoderr   r   nemo.collections.llm.peftr   nemo.collections.llm.recipesr
   nemo.lightningr   r   r   r   r   r   <module>   s   