o
    i:	                     @   s   d dl mZ d dlZd dlmZ d dlmZ ddlmZ ddlm	Z	 dd	l
mZ dd
lmZ ddlmZ ddlmZ G dd deZdS )    )OptionalN)Module)Tensor   )Errors   )AttentionMask)BertEmbeddings)PyTorchTransformerOutput   )AlbertConfig)AlbertLayerGroupc                	       s\   e Zd Zdef fddZdedefddZ		dd	ed
ee dee de	fddZ
  ZS )AlbertEncoderconfigc                    s   t     j| _ j| _ jj| _ jj}| j| dkr(tt	j
j| j|dt j j| _tj fddt|D | _d S )Nr   )num_hidden_layersnum_hidden_groupsc                    s   g | ]	}t  j jqS  )r   layer	attention).0_r   r   ^/home/ubuntu/.local/lib/python3.10/site-packages/curated_transformers/models/albert/encoder.py
<listcomp>&   s    z*AlbertEncoder.__init__.<locals>.<listcomp>)super__init__padding_idxmodel_max_lengthmax_seq_lenr   r   r   
ValueErrorr   E002formatr	   	embedding
embeddingstorchnn
ModuleListrangegroups)selfr   r   	__class__r   r   r      s$   



zAlbertEncoder.__init__xreturnc                 C   s   t || jdS )N)	bool_mask)r   ner   )r)   r,   r   r   r   _create_attention_mask,   s   z$AlbertEncoder._create_attention_maskN	input_idsattention_masktoken_type_idsc           
      C   st   |du r	|  |}| ||d}|}| jt| j }g }| jD ]}t|D ]}	|||d}|| q%qt||dS )zb
        Shapes:
            input_ids, attention_mask, token_type_ids - (batch, seq_len)
        N)	attn_mask)embedding_outputlayer_hidden_states)r0   r#   r   lenr(   r'   appendr
   )
r)   r1   r2   r3   r#   layer_outputlayers_per_grouplayer_outputsgroupr   r   r   r   forward/   s   


zAlbertEncoder.forward)NN)__name__
__module____qualname__r   r   r   r   r0   r   r
   r=   __classcell__r   r   r*   r   r      s     r   )typingr   r$   torch.nnr   r   errorsr   r   r   bert.embeddingsr	   outputr
   r   r   layer_groupr   r   r   r   r   r   <module>   s    