o
    i                     @   s|   d dl mZ d dlZd dlmZ d dlmZ ddlmZ ddlm	Z	 ddl
mZ d	d
lmZ d	dlmZ G dd deZdS )    )OptionalN)Module)Tensor   )
BertConfig)BertEmbeddings)BertEncoderLayer   )AttentionMask)PyTorchTransformerOutputc                	       s\   e Zd Zdef fddZdedefddZ		dd	ed
ee dee de	fddZ
  ZS )BertEncoderconfigc                    sR   t    t j j| _ j| _ j| _t	j
 fddt jjD | _d S )Nc                    s   g | ]	}t  j jqS  )r   layer	attention).0_r   r   \/home/ubuntu/.local/lib/python3.10/site-packages/curated_transformers/models/bert/encoder.py
<listcomp>   s    z(BertEncoder.__init__.<locals>.<listcomp>)super__init__r   	embeddingr   
embeddingspadding_idxmodel_max_lengthmax_seq_lentorchnn
ModuleListrangenum_hidden_layerslayers)selfr   	__class__r   r   r      s   



zBertEncoder.__init__xreturnc                 C   s   t || jdS )N)	bool_mask)r
   ner   )r#   r&   r   r   r   _create_attention_mask   s   z"BertEncoder._create_attention_maskN	input_idsattention_masktoken_type_idsc                 C   sT   |du r	|  |}| ||d}|}g }| jD ]}|||}|| qt||dS )zb
        Shapes:
            input_ids, attention_mask, token_type_ids - (batch, seq_len)
        N)embedding_outputlayer_hidden_states)r*   r   r"   appendr   )r#   r+   r,   r-   r   layer_outputlayer_outputsr   r   r   r   forward"   s   



zBertEncoder.forward)NN)__name__
__module____qualname__r   r   r   r
   r*   r   r   r3   __classcell__r   r   r$   r   r      s     r   )typingr   r   torch.nnr   r   r   r   r   r   r   r   r   r
   outputr   r   r   r   r   r   <module>   s    