o
    ´©io  ã                   @   sf   d Z ddlZddlmZ G dd„ dejjƒZG dd„ dejƒZG dd„ dejƒZG d	d
„ d
ejƒZdS )zLayer normalization module.é    Nc                       ó.   e Zd ZdZd‡ fdd„	Z‡ fdd„Z‡  ZS )Ú	LayerNormz€Layer normalization module.

    Args:
        nout (int): Output dim size.
        dim (int): Dimension to be normalized.

    éÿÿÿÿc                    s   t t| ƒj|dd || _dS )zConstruct an LayerNorm object.gê-™—q=)ÚepsN)Úsuperr   Ú__init__Údim)ÚselfÚnoutr   ©Ú	__class__© úX/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/transformer/layer_norm.pyr      s   
zLayerNorm.__init__c                    s>   | j dkrtt| ƒ |¡S tt| ƒ | | j d¡¡ | j d¡S )zžApply layer normalization.

        Args:
            x (torch.Tensor): Input tensor.

        Returns:
            torch.Tensor: Normalized tensor.

        r   )r   r   r   ÚforwardÚ	transpose©r	   Úxr   r   r   r      s   

$zLayerNorm.forward)r   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   Ú__classcell__r   r   r   r   r      s    r   c                       s*   e Zd ZdZd‡ fdd„	Zdd„ Z‡  ZS )	ÚGlobalLayerNormaP  Calculate Global Layer Normalization.

    Arguments
    ---------
       dim : (int or list or torch.Size)
           Input shape from an expected input of size.
       eps : float
           A value added to the denominator for numerical stability.
       elementwise_affine : bool
          A boolean value that when set to True,
          this module has learnable per-element affine parameters
          initialized to ones (for weights) and zeros (for biases).

    Example
    -------
    >>> x = torch.randn(5, 10, 20)
    >>> GLN = GlobalLayerNorm(10, 3)
    >>> x_norm = GLN(x)
    ç:Œ0âŽyE>Tc                    s¶   t t| ƒ ¡  || _|| _|| _| jrM|dkr-t t 	| jd¡¡| _
t t | jd¡¡| _|dkrKt t 	| jdd¡¡| _
t t | jdd¡¡| _d S d S |  dd ¡ |  dd ¡ d S )Né   é   é   ÚweightÚbias)r   r   r   r   r   Úelementwise_affineÚnnÚ	ParameterÚtorchÚonesr   Úzerosr   Úregister_parameter)r	   r   Úshaper   r    r   r   r   r   ?   s   þzGlobalLayerNorm.__init__c                 C   sö   |  ¡ dkr<tj|ddd}tj|| d ddd}| jr0| j||  t || j ¡ | j }n|| t || j ¡ }|  ¡ dkrytj|ddd}tj|| d ddd}| jrm| j||  t || j ¡ | j }|S || t || j ¡ }|S )z›Returns the normalized tensor.

        Arguments
        ---------
        x : torch.Tensor
            Tensor of size [N, C, K, S] or [N, C, L].
        r   )r   é   T)Úkeepdimr(   r   )r   r(   r   )r   r#   Úmeanr    r   Úsqrtr   r   )r	   r   r*   Úvarr   r   r   r   P   s   &$ÿzGlobalLayerNorm.forward)r   Tr   r   r   r   r   r   *   s    r   c                       r   )ÚCumulativeLayerNorma\  Calculate Cumulative Layer Normalization.

       Arguments
       ---------
       dim : int
        Dimension that you want to normalize.
       elementwise_affine : True
        Learnable per-element affine parameters.

    Example
    -------
    >>> x = torch.randn(5, 10, 20)
    >>> CLN = CumulativeLayerNorm(10)
    >>> x_norm = CLN(x)
    Tc                    s   t t| ƒj||dd d S )Nr   )r    r   )r   r-   r   )r	   r   r    r   r   r   r      s   

ÿzCumulativeLayerNorm.__init__c                    sx   |  ¡ dkr | dddd¡ ¡ }tƒ  |¡}| dddd¡ ¡ }|  ¡ dkr:t |dd¡}tƒ  |¡}t |dd¡}|S )z—Returns the normalized tensor.

        Arguments
        ---------
        x : torch.Tensor
            Tensor size [N, C, K, S] or [N, C, L]
        r   r   r(   r   r   )r   ÚpermuteÚ
contiguousr   r   r#   r   r   r   r   r   r   „   s   
zCumulativeLayerNorm.forward)Tr   r   r   r   r   r-   n   s    r-   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )Ú	ScaleNormçñhãˆµøä>c                    s0   t ƒ  ¡  |d | _|| _t t d¡¡| _d S )Ng      à¿r   )	r   r   Úscaler   r!   r"   r#   r$   Úg)r	   r   r   r   r   r   r   ž   s   

zScaleNorm.__init__c                 C   s.   t j|ddd| j }||j| jd | j S )Nr   T)r   r)   )Úmin)r#   Únormr2   Úclampr   r3   )r	   r   r5   r   r   r   r   ¤   s   zScaleNorm.forward)r1   )r   r   r   r   r   r   r   r   r   r   r0      s    r0   )	r   r#   Útorch.nnr!   r   ÚModuler   r-   r0   r   r   r   r   Ú<module>   s   D/