o
    }oi"                     @   s  d dl Z d dlZd dlmZmZmZ G dd dZG dd dZe jj	ej
  ddG d	d
 d
Zd dlmZmZ d dl Z d dlZd dlmZ dd Zdd Zd dlmZ d dl Z d dlZd dlmZ dd Zd dlmZmZ d dl Z d dlmZ d dlmZ dd Zdd ZdS )    N)BertEmbeddingHeadBertEmbeddingLargeConfigBertEmbeddingMiniConfigc                   @   s*   e Zd Zejdd Zdd Zdd ZdS )TestBertEmbeddingHeadc                 C   s
   t ddS )N   )word_embedding_dimension)r   self r
   c/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/bert/model/test_embedding.pyembedding_head   s   
z$TestBertEmbeddingHead.embedding_headc           	      C   sp   d}d}d}t |||}t ||}|||}|j||fks!J t j|ddd}t j|t |dds6J d S )N      r      )pdimgư>)atol)torchrandnonesshapenormallclose	ones_like)	r	   r   
batch_size
seq_length
hidden_dimtoken_embeddingsattention_maskoutputnormsr
   r
   r   test_embedding_head_forward   s   
z1TestBertEmbeddingHead.test_embedding_head_forwardc                 C   sV   d}d}d}t |||}t jg dg dgt jd}|||}|j||fks)J d S )Nr   r   r   )r   r   r   r   )r   r   r   r   )dtype)r   r   tensorfloatr   )r	   r   r   r   r   r   r   r   r
   r
   r   !test_embedding_head_masked_tokens,   s   
z7TestBertEmbeddingHead.test_embedding_head_masked_tokensN)__name__
__module____qualname__pytestfixturer   r!   r%   r
   r
   r
   r   r      s
    
r   c                   @   s   e Zd Zdd Zdd ZdS )TestBertEmbeddingConfigc                 C   B   t  }|jdks
J |jdksJ |jdksJ |jdksJ d S )N   i   i      )r   
num_layershidden_sizeintermediate_sizenum_attention_headsr	   configr
   r
   r   test_large_config:   
   z)TestBertEmbeddingConfig.test_large_configc                 C   r,   )N   i  i      )r   r/   r0   r1   r2   r3   r
   r
   r   test_mini_configA   r6   z(TestBertEmbeddingConfig.test_mini_configN)r&   r'   r(   r5   r9   r
   r
   r
   r   r+   9   s    r+   zTest requires GPU)reasonc                   @   s6   e Zd Zejdd Zejdd Zejdd ZdS )TestBertEmbeddingModelc                 C   s   t  }d|_|S )Ni:w  )	MagicMock
vocab_size)r	   	tokenizerr
   r
   r   mock_tokenizerK   s   z%TestBertEmbeddingModel.mock_tokenizerc                 C   s   t  S N)r   r   r
   r
   r   model_configQ   s   z#TestBertEmbeddingModel.model_configc                 C   s   t ||d}|  |S N)r4   r>   )BertEmbeddingModelconfigure_model)r	   rA   r?   modelr
   r
   r   rE   U   s   zTestBertEmbeddingModel.modelN)r&   r'   r(   r)   r*   r?   rA   rE   r
   r
   r
   r   r;   I   s    

r;   )r<   patch)bert_embedding_data_stepc               
      sz  d} d}t dd| |ft | |t | |t ddgd t g}d fdd		}  D ]}t|t jr=t	|d
|_
q/tddd: tddd d
" tddd t|}W d    n1 scw   Y  W d    n1 srw   Y  W d    n1 sw   Y  d|v sJ d|v sJ d|v sJ d|v sJ  d j
   d j
   d j
  |d d u sJ d S )Nr   
   r     r   )	input_idsr   token_type_idslabelsTc                       t  d S Nr   r   r   non_blocking
mock_batchr
   r   	mock_cudas      z0test_bert_embedding_data_step.<locals>.mock_cudaside_effect4megatron.core.parallel_state.is_pipeline_first_stagereturn_value-megatron.core.utils.get_batch_on_this_cp_rankc                 S      | S r@   r
   xr
   r
   r   <lambda>}       z/test_bert_embedding_data_step.<locals>.<lambda><megatron.core.parallel_state.get_context_parallel_world_sizerJ   r   rK   rL   T)r   randintr   zerosr#   itervalues
isinstanceTensorr<   cudarF   rG   assert_called_once)r   r   mock_iteratorrT   r#   resultr
   rR   r   test_bert_embedding_data_stepd   s<   



rm   c               
      s2  d} d}t dd| |ft | |t | |d t d d fg}d fdd	}  D ]}t|t jr:t|d	|_	q,t
d
dd: t
ddd d	" t
ddd t|W d    n1 s`w   Y  W d    n1 sow   Y  W d    n1 s~w   Y  ttsJ tfdddD sJ d S )Nr   rH   r   rI   )rJ   r   rK   Tc                    rM   rN   rO   rP   rR   r
   r   rT      rU   z<test_bert_embedding_data_step_tuple_input.<locals>.mock_cudarV   rX   rY   r[   c                 S   r\   r@   r
   r]   r
   r
   r   r_      r`   z;test_bert_embedding_data_step_tuple_input.<locals>.<lambda>ra   r   c                 3   s    | ]}| v V  qd S r@   r
   ).0key)rl   r
   r   	<genexpr>   s    z<test_bert_embedding_data_step_tuple_input.<locals>.<genexpr>rb   )r   rc   r   rd   re   rf   rg   rh   r<   ri   rF   rG   dictall)r   r   rk   rT   r#   r
   )rS   rl   r   )test_bert_embedding_data_step_tuple_input   s.   


rs   )r<   )bert_embedding_forward_stepc                  C   sz   d} d}d}t dd| |ft | |t | |t | d}t }t |_d|j_t | |}t|d|_t	|| d S )Nr   rH   r   r   rI   )rJ   r   rK   	extra_keyrY   )
r   rc   r   rd   r<   r4   num_tokentypesr   __call__rt   )r   r   r0   batch
mock_modelexpected_outputr
   r
   r    test_bert_embedding_forward_step   s   

r{   ),BERTInBatchExclusiveHardNegativesRankingLoss)rC   c                  C   L   t  } t| t d}|j}t|tsJ |j| jksJ |j| jks$J d S rB   )	r   rC   r<   training_loss_reductionrg   r|   num_hard_negativesscalece_loss_scaler4   rE   loss_reductionr
   r
   r   +test_training_loss_reduction_initialization      r   c                  C   r}   rB   )	r   rC   r<   validation_loss_reductionrg   r|   r   r   r   r   r
   r
   r   -test_validation_loss_reduction_initialization   r   r   )r)   r   )nemo.collections.llm.bert.model.embeddingr   r   r   r   r+   markskipifri   is_availabler;   unittest.mockr<   rF   rG   rm   rs   rt   r{   nemo.collections.llm.bert.lossr|   rC   r   r   r
   r
   r
   r   <module>   s0   !,