o
    }o™i—  ã                   @   s´   d dl mZmZmZmZ dd„ Zdd„ Zdd„ Zdd	„ Zd d
l	m
Z
 d dlmZ d dlZd dlZd dl mZmZmZmZmZmZmZmZmZmZ e
G dd„ dƒƒZG dd„ dƒZdS )é    )ÚHuggingFaceBertBaseConfigÚHuggingFaceBertLargeConfigÚMegatronBertBaseConfigÚMegatronBertLargeConfigc                  C   óP   t ƒ } | jdks
J ‚| jdksJ ‚| jdksJ ‚| jdksJ ‚| jdks&J ‚d S )NÚhuggingfaceé   é   é   )r   Ú	bert_typeÚ
num_layersÚhidden_sizeÚffn_hidden_sizeÚnum_attention_heads©Úconfig© r   ú^/home/ubuntu/.local/lib/python3.10/site-packages/tests/collections/llm/bert/model/test_bert.pyÚ!test_huggingface_bert_base_config   ó   r   c                  C   óP   t ƒ } | jdks
J ‚| jdksJ ‚| jdksJ ‚| jdksJ ‚| jdks&J ‚d S )Nr   é   é   é   é   )r   r   r   r   r   r   r   r   r   r   Ú"test_huggingface_bert_large_config    r   r   c                  C   r   )NÚmegatronr   r	   r
   )r   r   r   r   r   r   r   r   r   r   Útest_megatron_bert_base_config)   r   r   c                  C   r   )Nr   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   Útest_megatron_bert_large_config2   r   r   ©Ú	dataclass)Ú	MagicMockN)
Ú_export_embeddingÚ_export_qkvÚ_export_qkv_biasÚ_import_embeddingÚ_import_embedding_2Ú_import_output_biasÚ_import_qkvÚ_import_qkv_2Ú_import_qkv_biasÚ_import_qkv_bias_2c                   @   óJ   e Zd ZU dZeed< dZeed< dZeed< dZeed< d	Z	eed
< dS )Ú
MockConfigr   r   r	   r   é@   Úkv_channelsé€   Úmake_vocab_size_divisible_byé:w  Ú
vocab_sizeN©
Ú__name__Ú
__module__Ú__qualname__r   ÚintÚ__annotations__r   r/   r1   r3   r   r   r   r   r-   O   ó   
 r-   c                   @   sF   e Zd Zejdd„ ƒZddlmZ eG dd„ dƒƒZG dd „ d ƒZ	dS )	ÚTestBertTransformsc                 C   ó   t ƒ }tƒ |j_tƒ |j_|S ©N©r!   r-   Útargetr   Úsource©ÚselfÚctxr   r   r   Úmock_ctxY   ó   

zTestBertTransforms.mock_ctxr   r   c                   @   r,   )zTestBertTransforms.MockConfigr   r   r	   r   r.   r/   r0   r1   r2   r3   Nr4   r   r   r   r   r-   b   r:   r-   c                   @   sR   e Zd Zejdd„ ƒZdd„ Zdd„ Zdd„ Zd	d
„ Z	dd„ Z
dd„ Zdd„ ZdS )z%TestBertTransforms.TestBertTransformsc                 C   r<   r=   r>   rA   r   r   r   rD   k   rE   z.TestBertTransforms.TestBertTransforms.mock_ctxc                 C   s~   d}d}d}t  || |¡}t  || |¡}t  || |¡}tjtjfD ]}|||||ƒ}	d| | |f}
|	j|
ks<J ‚q$d S ©Nr	   r.   r   é   )ÚtorchÚrandnr(   Ú	transformr)   Úshape)rB   rD   r   Ú	head_sizeÚ	num_headsÚqÚkÚvÚtransform_fnÚresultÚexpected_shaper   r   r   Útest_import_qkvr   s   ûz5TestBertTransforms.TestBertTransforms.test_import_qkvc           
      C   sr   d}d}t  || ¡}t  || ¡}t  || ¡}tjtjfD ]}|||||ƒ}d| | f}	|j|	ks6J ‚qd S ©Nr.   r   rG   )rH   rI   r*   rJ   r+   rK   )
rB   rD   rL   rM   ÚqbÚkbÚvbrQ   rR   rS   r   r   r   Útest_import_qkv_biasƒ   s   ûz:TestBertTransforms.TestBertTransforms.test_import_qkv_biasc                 C   s’   d}d}t  ||¡}tjtjfD ]6}|||ƒ}tt  t  |¡d ¡d ƒ}|j||fks.J ‚t j	 
|d |… |¡ t  ||d … dk¡sFJ ‚qd S )Né0u  r	   r0   r   )rH   rI   r%   rJ   r&   r8   ÚceilÚtensorrK   ÚtestingÚassert_closeÚall)rB   rD   r3   r   Ú	embeddingrQ   rR   Úexpected_padded_sizer   r   r   Útest_import_embedding“   s   
÷z;TestBertTransforms.TestBertTransforms.test_import_embeddingc                 C   sz   d}t  |¡}t ||¡}tt  t  |¡d ¡d ƒ}|j|fks#J ‚t j 	|d |… |¡ t  
||d … dk¡s;J ‚d S )NrZ   r0   r   )rH   rI   r'   rJ   r8   r[   r\   rK   r]   r^   r_   )rB   rD   r3   ÚbiasrR   ra   r   r   r   Útest_import_output_bias¤   s   
z=TestBertTransforms.TestBertTransforms.test_import_output_biasc           	      C   sx   d}d}d}t  d| | |¡}t ||¡\}}}|j|| |fks$J ‚|j|| |fks/J ‚|j|| |fks:J ‚d S rF   )rH   rI   r#   rJ   rK   )	rB   rD   r   rL   rM   Ú
linear_qkvÚq_projÚk_projÚv_projr   r   r   Útest_export_qkv²   s   z5TestBertTransforms.TestBertTransforms.test_export_qkvc                 C   sl   d}d}t  d| | ¡}t ||¡\}}}|j|| fks J ‚|j|| fks*J ‚|j|| fks4J ‚d S rU   )rH   rI   r$   rJ   rK   )rB   rD   rL   rM   Úqkv_biasÚq_biasÚk_biasÚv_biasr   r   r   Útest_export_qkv_biasÁ   s   z:TestBertTransforms.TestBertTransforms.test_export_qkv_biasc                 C   sP   d}d}d}t  ||¡}t ||¡}|j||fksJ ‚t j ||d |… ¡ d S )Nr2   r	   i x  )rH   rI   r"   rJ   rK   r]   r^   )rB   rD   r3   r   Úpadded_vocab_sizer`   rR   r   r   r   Útest_export_embeddingÏ   s   z;TestBertTransforms.TestBertTransforms.test_export_embeddingN)r5   r6   r7   ÚpytestÚfixturerD   rT   rY   rb   rd   ri   rn   rp   r   r   r   r   r;   j   s    
N)
r5   r6   r7   rq   rr   rD   Údataclassesr    r-   r;   r   r   r   r   r;   X   s    
r;   )Ú$nemo.collections.llm.bert.model.bertr   r   r   r   r   r   r   r   rs   r    Úunittest.mockr!   rq   rH   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r-   r;   r   r   r   r   Ú<module>   s   				0