o
    }oi[                     @   sb   d dl Zd dlZd dlZd dlmZmZmZ d dlm	Z	 d dl
mZ d dlmZ G dd dZdS )    N)BertEmbeddingLargeConfigBertEmbeddingMiniConfigBertEmbeddingModel)bert_embedding)Trainer)TimingCallbackc                   @   sD   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dS )TestBertEmbeddingc                 C   N   t d}t|tjsJ |jtksJ t|jtjsJ |jjtks%J d S )N	bert_110m)	r   bert_embedding_model
isinstancerunConfig__fn_or_cls__r   configr   selfmodel_config r   e/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/recipes/test_bert_embedding.pytest_bert_embedding_model_110m   
   
z0TestBertEmbedding.test_bert_embedding_model_110mc                 C   r	   )N	bert_340m)	r   r   r   r   r   r   r   r   r   r   r   r   r   test_bert_embedding_model_340m"   r   z0TestBertEmbedding.test_bert_embedding_model_340mc                 C   s>   t jtdd td W d    d S 1 sw   Y  d S )Nz%Invalid BERT version: invalid_version)matchinvalid_version)pytestraisesAssertionErrorr   r   )r   r   r   r   )test_bert_embedding_model_invalid_version)   s   "z;TestBertEmbedding.test_bert_embedding_model_invalid_versionc                 C   s   t  }t|tjsJ |jtksJ |jjdksJ |jj	dks#J |jj
d u s+J |jjd u s3J |jjdks;J |jjdu sCJ |jdksJJ |jdksQJ |jdksXJ |jdks_J |jdksfJ |jdksmJ d S )N      Fi{     
   i  )r   bert_trainerr   r   r   r   r   strategytensor_model_parallel_sizepipeline_model_parallel_sizepipeline_dtype$virtual_pipeline_model_parallel_sizecontext_parallel_sizesequence_parallel	max_stepsaccumulate_grad_batcheslimit_test_batcheslimit_val_batcheslog_every_n_stepsval_check_intervalr   trainerr   r   r   "test_bert_trainer_default_settings-   s   z4TestBertEmbedding.test_bert_trainer_default_settingsc                 C   s   t jddtjddddddddddddd	}|jjdksJ |jjdks%J |jjtjks.J |jjdks6J |jj	dks>J |jj
du sFJ |jdksMJ |jdksTJ |jdks[J |jdksbJ |jdksiJ |jdkspJ |jdkswJ |jdks~J d S )
N   r    Ti  z16-mixed@      i  )tensor_parallelismpipeline_parallelismpipeline_parallelism_typevirtual_pipeline_parallelismcontext_parallelismsequence_parallelism	num_nodesnum_gpus_per_noder,   	precisionr-   r.   r/   r0   r1   )r   r$   torchfloat16r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r>   devicesr2   r   r   r   !test_bert_trainer_custom_settingsB   s>   z3TestBertEmbedding.test_bert_trainer_custom_settingsc                 C   s*   t tg}tj|d}|j|ksJ d S )N)	callbacks)r   r   r   r   r$   rE   )r   rE   r3   r   r   r    test_bert_trainer_with_callbacksg   s   z2TestBertEmbedding.test_bert_trainer_with_callbacksc                 C   sf   t  }|jjjdu sJ |jjjdu sJ |jjjdu sJ |jjjdu s(J |jjjdu s1J d S )NTF)	r   r$   r%   ddpcheck_for_nan_in_gradgrad_reduce_in_fp32overlap_grad_reduceoverlap_param_gatheraverage_in_collectiver2   r   r   r   test_bert_trainer_ddp_settingsl   s   z0TestBertEmbedding.test_bert_trainer_ddp_settingsN)
__name__
__module____qualname__r   r   r   r4   rD   rF   rM   r   r   r   r   r      s    %r   )nemo_runr   r   rA   nemo.collections.llmr   r   r   nemo.collections.llm.recipesr   nemo.lightningr   nemo.utils.exp_managerr   r   r   r   r   r   <module>   s   