o
    }oi                     @   s   d dl mZ d dlmZ d dlmZmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d d	lmZmZmZmZmZ d
dgZejjejjdZeG dd dZG dd de
ZG dd
 d
e
ZdS )    )contextmanager)	dataclass)DictOptionalN)nn)
Classifier)MultiLayerPerceptron)	typecheck)ChannelType	FloatType
LogitsTypeLogprobsType
NeuralTypeBertPretrainingTokenClassifierTokenClassifier)gelureluc                   @   sJ   e Zd ZU dZeed< dZeed< dZe	ed< dZ
eed< dZe	ed	< d
S )TokenClassifierConfig   
num_layersr   
activationTlog_softmax        dropoutuse_transformer_initN)__name__
__module____qualname__r   int__annotations__r   strr   boolr   floatr    r#   r#   j/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/asr/parts/submodules/token_classifier.pyr      s   
 r   c                       s   e Zd ZdZedeeef fddZedeeef fddZ						
		dde
de
de
dededededdf fddZedefddZededd fddZe dejdejfddZ  ZS )r   z`
    A module to perform token level classification tasks such as Named entity recognition.
    returnc                 C      dt dt iS Nhidden_states)BTDr   r
   selfr#   r#   r$   input_types-      zTokenClassifier.input_typesc                 C   (   | j jsdtdt iS dtdt iS z=
        Returns definitions of module output ports.
        logits)r)   r*   C	log_probsmlpr   r   r   r   r-   r#   r#   r$   output_types3      zTokenClassifier.output_typesr   r   Tr   hidden_sizenum_classesr   r   r   r   r   Nc                    s4   t  j||d t|||||d| _| j|d dS )q  
        Initializes the Token Classifier module.

        Args:
            hidden_size: the size of the hidden dimension
            num_classes: number of classes
            num_layers: number of fully connected layers in the multilayer perceptron (MLP)
            activation: activation to usee between fully connected layers in the MLP
            log_softmax: whether to apply softmax to the output of the MLP
            dropout: dropout to apply to the input hidden states
            use_transformer_init: whether to initialize the weights of the classifier head with the same approach used in Transformer
        r:   r   r   r   r   r   N)super__init__r   r7   	post_initr.   r:   r;   r   r   r   r   r   	__class__r#   r$   rA   =   s
   
zTokenClassifier.__init__c                 C      | j jS Nr7   r   r-   r#   r#   r$   r   Y      zTokenClassifier.log_softmaxvaluec                 c   $    | j j}|| j _| V  || j _d S rG   rH   r.   rJ   prevr#   r#   r$   with_log_softmax_enabled]   
   z(TokenClassifier.with_log_softmax_enabledr(   c                 C   s   |  |}| |}|S a3  
        Performs the forward step of the module.
        Args:
            hidden_states: batch of hidden states (for example, from the BERT encoder module)
                [BATCH_SIZE x SEQ_LENGTH x HIDDEN_SIZE]
        Returns: logits value for each class [BATCH_SIZE x SEQ_LENGTH x NUM_CLASSES]
        )r   r7   )r.   r(   r3   r#   r#   r$   forwardd   s   
	
zTokenClassifier.forwardr   r   Tr   T)r   r   r   __doc__propertyr   r    r   r/   r8   r   r!   r"   rA   r   r   rN   r	   torchTensorrQ   __classcell__r#   r#   rD   r$   r   (   sD    	 c                       s   e Zd ZdZedeeef fddZede	eeef  fddZ
					
		ddedededededededdf fddZedefddZededdfddZe dejdejfddZ  ZS )r   zT
    A module to perform token level classification tasks for Bert pretraining.
    r%   c                 C   r&   r'   r,   r-   r#   r#   r$   r/   w   r0   z*BertPretrainingTokenClassifier.input_typesc                 C   r1   r2   r6   r-   r#   r#   r$   r8   }   r9   z+BertPretrainingTokenClassifier.output_typesr   r   Tr   r:   r;   r   r   r   r   r   Nc                    st   t  j||d |tvrtd| dt||| _t| | _tj|dd| _	t
|||||d| _| j|d dS )	r<   r=   zactivation "z" not foundg-q=)epsr>   r?   N)r@   rA   ACT2FN
ValueErrorr   Lineardenseact	LayerNormnormr   r7   rB   rC   rD   r#   r$   rA      s   

z'BertPretrainingTokenClassifier.__init__c                 C   rF   rG   rH   r-   r#   r#   r$   r      rI   z*BertPretrainingTokenClassifier.log_softmaxrJ   r   c                 c   rK   rG   rH   rL   r#   r#   r$   rN      rO   z7BertPretrainingTokenClassifier.with_log_softmax_enabledr(   c                 C   s6   |  |}| |}| |}| |}| |}|S rP   )r   r\   r]   r_   r7   )r.   r(   	transformr3   r#   r#   r$   rQ      s   
	



z&BertPretrainingTokenClassifier.forwardrR   )r   r   r   rS   rT   r   r    r   r/   r   r8   r   r!   r"   rA   r   r   rN   r	   rU   rV   rQ   rW   r#   r#   rD   r$   r   r   sD    	" )
contextlibr   dataclassesr   typingr   r   rU   r   0nemo.collections.asr.parts.submodules.classifierr   nemo.collections.common.partsr   nemo.core.classesr	   nemo.core.neural_typesr
   r   r   r   r   __all__
functionalr   r   rY   r   r   r   r#   r#   r#   r$   <module>   s   J