o
    }oi                     @  sl   d dl mZ d dlmZ d dlmZ d dlZd dlmZ G dd dej	Z
dddZdddZdddZdS )    )annotations)reduce)ListNc                      s:   e Zd ZdZd fddZedd	d
ZdddZ  ZS )FusedBatchNorm1dz
    Fused BatchNorm to use in Conformer to improve accuracy in finetuning with TTS scenario
    Drop-in replacement for BatchNorm1d with simple affine projection
    num_featuresintc                   s8   t    || _tt|| _tt|| _	dS )zl
        Args:
            num_features: number of channels, see original BatchNorm1d documentation
        N)
super__init__r   nn	Parametertorchonesweightzerosbias)selfr   	__class__ c/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/asr/parts/submodules/batchnorm.pyr	      s   
zFusedBatchNorm1d.__init__bnnn.BatchNorm1dreturnc                 C  s^   t |tjsJ t|j}t|jj|j	 }|j
j| |j
_|jj|jj|j
j  |j_|S )a  
        Construct FusedBatchNorm1d module from BatchNorm1d
        Args:
            bn: original BatchNorm module

        Returns:
            FusedBatchNorm1d module with initialized params; in eval mode result is equivalent to original BatchNorm
        )
isinstancer
   BatchNorm1dr   r   r   sqrtrunning_vardataepsr   r   running_mean)clsr   fused_bnstdr   r   r   from_batchnorm(   s   

zFusedBatchNorm1d.from_batchnormxtorch.Tensorc                 C  sH   |  dkr|| jd | jd S |  dksJ || j | j S )N      )dimr   	unsqueezer   )r   r$   r   r   r   forward;   s   zFusedBatchNorm1d.forward)r   r   )r   r   r   r   )r$   r%   )	__name__
__module____qualname____doc__r	   classmethodr#   r+   __classcell__r   r   r   r   r      s    
r   module	nn.Modulefull_layer_namestrr   c                 C  s   |j dd}tt|| S )N.)sep)splitr   getattr)r2   r4   namesr   r   r   _get_module_by_nameB   s   r;   c                 C  st   t | |}t|tjsJ t|}z|jddd\}}tt | ||| W dS  ty9   |}t| || Y dS w )z
    Replace BatchNorm1d named `full_layer_name` in nn.Module with FusedBatchNorm1d
    Args:
        module: nn.Module instance, modified inplace
        full_layer_name: name of BatchNorm1d submodule in module to replace
    r6      )maxsplitN)	r;   r   r
   r   r   r#   rsplitsetattr
ValueError)r2   r4   r   r!   parent_name	norm_namer   r   r   replace_bn_with_fused_bnG   s   

rC   model	List[str]c                 C  s:   g }|   D ]\}}t|tjrt| | || q|S )z
    Replace BatchNorm1d with FusedBatchNorm1d in model
    Args:
        model: nn.Module instance, modified inplace

    Returns:
        list of replaced module names
    )named_modulesr   r
   r   rC   append)rD   replaced_module_namesnamer2   r   r   r   replace_bn_with_fused_bn_allY   s   	

rJ   )r2   r3   r4   r5   r   r3   )r2   r3   r4   r5   )rD   r3   r   rE   )
__future__r   	functoolsr   typingr   r   torch.nnr
   Moduler   r;   rC   rJ   r   r   r   r   <module>   s   
*
