o
    i@
                     @   s`   d dl mZ d dlZd dlmZ d dlmZmZmZmZm	Z	 ddl
mZmZ G dd de	ZdS )	    )OptionalN)Tensor)Dropout	Embedding	LayerNormLinearModule   )BertEmbeddingConfigBertLayerConfigc                	       sv   e Zd Zdededdf fddZdedefdd	Zdedefd
dZ		ddede	e de	e defddZ
  ZS )BertEmbeddingsembedding_configlayer_configreturnNc                    s   t    t|j|jd| _t|j|jd| _t|j|jd| _	|j|j
kr/t|j|j
| _nd | _t|j|jd| _t|jd| _d S )N)num_embeddingsembedding_dim)eps)p)super__init__r   
vocab_sizeembedding_widthword_embeddingstype_vocab_sizetoken_type_embeddingsmax_position_embeddingsposition_embeddingshidden_widthr   
projectionr   layer_norm_eps
layer_normr   dropout_probdropout)selfr   r   	__class__ _/home/ubuntu/.local/lib/python3.10/site-packages/curated_transformers/models/bert/embeddings.pyr   
   s,   
zBertEmbeddings.__init__xc                 C   s   t j|jd |jdddS )Nr	   )device)torcharangeshaper)   expandr#   r(   r&   r&   r'   _get_position_ids(   s   z BertEmbeddings._get_position_idsc                 C   s
   t |S )N)r+   
zeros_liker/   r&   r&   r'   _get_token_type_ids+   s   
z"BertEmbeddings._get_token_type_ids	input_idstoken_type_idsposition_idsc           
      C   s   |du r	|  |}|du r| |}| |}| |}| |}|}||7 }||7 }| |}| |}	| jdur?| |	S |	S )z`
        Shapes:
            input_ids, token_type_ids, position_ids - (batch, seq_len)
        N)r2   r0   r   r   r   r    r"   r   )
r#   r3   r4   r5   input_embeddingsr   r   embedding_sum
normalized
embeddingsr&   r&   r'   forward.   s   









zBertEmbeddings.forward)NN)__name__
__module____qualname__r
   r   r   r   r0   r2   r   r:   __classcell__r&   r&   r$   r'   r   	   s*    r   )typingr   r+   r   torch.nnr   r   r   r   r   configr
   r   r   r&   r&   r&   r'   <module>   s    