o
    ϯiC                     @   s  U d dl Z d dlZd dlmZmZmZmZmZmZm	Z	 d dl
Z
d dlZd dl
mZ e
jje
jje
jje
jjfZee	ej  ed< eeZejed< G dd dZG dd	 d	Ze
 	
	ddejdee dedee ddf
ddZdejdeej fddZdS )    N)AnyDictIterableListOptionalTupleType)nnBN_MODULE_TYPESloggerc                   @   sH   e Zd ZdZdejdejddfddZdejd	ejd
eddfddZdS )_MeanOfBatchVarianceEstimatora  
    Note that PyTorch's running_var means "running average of
    bessel-corrected batch variance". (PyTorch's BN normalizes by biased
    variance, but updates EMA by unbiased (bessel-corrected) variance).
    So we estimate population variance by "simple average of bessel-corrected
    batch variance". This is the same as in the BatchNorm paper, Sec 3.1.
    This estimator converges to population variance as long as batch size
    is not too small, and total #samples for PreciseBN is large enough.
    Its convergence is affected by small batch size.

    In this implementation, we also don't distinguish differences in batch size.
    We assume every batch contributes equally to the population statistics.
    mean_buffer
var_bufferreturnNc                 C   "   t || _t || _d| _d S Nr   )torch
zeros_likepop_meanpop_varindselfr   r    r   H/home/ubuntu/.local/lib/python3.10/site-packages/fvcore/nn/precise_bn.py__init__)      
z&_MeanOfBatchVarianceEstimator.__init__
batch_mean	batch_var
batch_sizec                 C   sF   |  j d7  _ |  j|| j | j  7  _|  j|| j | j  7  _d S N   )r   r   r   )r   r   r   r   r   r   r   update.   s   z$_MeanOfBatchVarianceEstimator.update)	__name__
__module____qualname____doc__r   Tensorr   intr"   r   r   r   r   r      s    r   c                   @   s\   e Zd ZdZdejdejddfddZdejd	ejd
eddfddZe	dejfddZ
dS )_PopulationVarianceEstimatoraU  
    Alternatively, one can estimate population variance by the sample variance
    of all batches combined. This needs to use the batch size of each batch
    in this function to undo the bessel-correction.
    This produces better estimation when each batch is small.
    See Appendix of the paper "Rethinking Batch in BatchNorm" for details.

    In this implementation, we also take into account varying batch sizes.
    A batch of N1 samples with a mean of M1 and a batch of N2 samples with a
    mean of M2 will produce a population mean of (N1M1+N2M2)/(N1+N2) instead
    of (M1+M2)/2.
    r   r   r   Nc                 C   r   r   )r   r   r   pop_square_meantotr   r   r   r   r   D   r   z%_PopulationVarianceEstimator.__init__r   r   r   c                 C   sf   |  j |7  _ | ||d |   }|  j|| j || j   7  _|  j|| j || j   7  _d S r    )r+   squarer   r*   )r   r   r   r   batch_square_meanr   r   r   r"   I   s   
z#_PopulationVarianceEstimator.updatec                 C   s   | j | j  S )N)r*   r   r,   )r   r   r   r   r   U   s   z$_PopulationVarianceEstimator.pop_var)r#   r$   r%   r&   r   r'   r   r(   r"   propertyr   r   r   r   r   r)   6   s    
r)      modeldata_loader	num_itersprogressr   c                    sz  t | }t|dkrdS tdt| d dd |D }|D ]}d|_q i  dtjd	ttj	 d
ttj	 f fddfdd|D }dd |D }d}	t
j
t||||dkdD ]-}
|	d7 }	   | |
 t|D ]\}} |d}|du r}qn|| |j|j| qnq\|	|d ksJ d||	t|D ]\}}|| j|_|| j|_|| |_q|D ]}|  qdS )a  
    Recompute and update the batch norm stats to make them more precise. During
    training both BN stats and the weight are changing after every iteration, so
    the running average can not precisely reflect the actual stats of the
    current model.
    In this function, the BN stats are recomputed with fixed weights, to make
    the running average more precise. Specifically, it computes the true average
    of per-batch mean/variance instead of the running average.
    See Sec. 3 of the paper "Rethinking Batch in BatchNorm" for details.

    Args:
        model (nn.Module): the model whose bn stats will be recomputed.

            Note that:

            1. This function will not alter the training mode of the given model.
               Users are responsible for setting the layers that needs
               precise-BN to training mode, prior to calling this function.

            2. Be careful if your models contain other stateful layers in
               addition to BN, i.e. layers whose state can change in forward
               iterations.  This function will alter their state. If you wish
               them unchanged, you need to either pass in a submodule without
               those layers, or backup the states.
        data_loader (iterator): an iterator. Produce data as inputs to the model.
        num_iters (int): number of iterations to compute the stats.
        progress: None or "tqdm". If set, use tqdm to report the progress.
    r   Nz$Computing precise BN statistics for z BN layers ...c                 S   s   g | ]}|j qS r   )momentum.0bnr   r   r   
<listcomp>   s    z#update_bn_stats.<locals>.<listcomp>g      ?moduleinputr   c                    sR   |  vsJ d|d }t |tjsJ d| | |jd  }| | < |fS )NzJSome BN layers are reused. This is not supported and probably not desired.r   z*BN layer should take tensor as input. Got r!   )
isinstancer   r'   numelshape)r9   r:   xr   )batch_size_per_bn_layerr   r   get_bn_batch_size_hook   s   
z/update_bn_stats.<locals>.get_bn_batch_size_hookc                    s   g | ]}|  qS r   )register_forward_pre_hookr5   )r@   r   r   r8      s    
c                 S   s   g | ]	}t |j|jqS r   )r)   running_meanrunning_varr5   r   r   r   r8      s    tqdm)totaldisabler!   z]update_bn_stats is meant to run for {} iterations, but the dataloader stops at {} iterations.)get_bn_moduleslenr   infor4   r	   Moduler   r   r'   rE   	itertoolsisliceclear	enumerategetr"   rB   rC   formatr   r   remove)r0   r1   r2   r3   	bn_layersmomentum_actualr7   hooks_to_remove
estimatorsr   inputsir   hookr   )r?   r@   r   update_bn_statsZ   s^   #



rZ   c                 C   s   dd |   D }|S )a@  
    Find all BatchNorm (BN) modules that are in training mode. See
    fvcore.precise_bn.BN_MODULE_TYPES for a list of all modules that are
    included in this search.

    Args:
        model (nn.Module): a model possibly containing BN modules.

    Returns:
        list[nn.Module]: all BN modules in the model.
    c                 S   s    g | ]}|j rt|tr|qS r   )trainingr;   r
   )r6   mr   r   r   r8      s    z"get_bn_modules.<locals>.<listcomp>)modules)r0   rS   r   r   r   rH      s   rH   )r/   N)rL   loggingtypingr   r   r   r   r   r   r   r   rE   r	   BatchNorm1dBatchNorm2dBatchNorm3dSyncBatchNormr
   rK   __annotations__	getLoggerr#   r   Loggerr   r)   no_gradr(   strrZ   rH   r   r   r   r   <module>   s:   
$$h