o
    پiX                     @   s$   d Z ddlZd
ddZddd	ZdS )aP   Adaptive Gradient Clipping

An impl of AGC, as per (https://arxiv.org/abs/2102.06171):

@article{brock2021high,
  author={Andrew Brock and Soham De and Samuel L. Smith and Karen Simonyan},
  title={High-Performance Large-Scale Image Recognition Without Normalization},
  journal={arXiv preprint arXiv:},
  year={2021}
}

Code references:
  * Official JAX impl (paper authors): https://github.com/deepmind/deepmind-research/tree/master/nfnets
  * Phil Wang's PyTorch gist: https://gist.github.com/lucidrains/0d6560077edac419ab5d3aa29e674d5c

Hacked together by / Copyright 2021 Ross Wightman
    N       @c                 C   s0   | j dkr
| |S | j|ttd| j ddS )N   T)dimkeepdim)ndimnormtuplerange)x	norm_type r   B/home/ubuntu/.local/lib/python3.10/site-packages/timm/utils/agc.pyunitwise_norm   s   

r   {Gz?MbP?c                 C   s   t | tjr	| g} | D ]?}|jd u rq| }|j }t||dj|d|}t||d}|||jdd  }	t	||k ||	}
|j 
|
 qd S )N)r   )mingư>)
isinstancetorchTensorgraddetachr   clamp_mul_clampwherecopy_)
parametersclip_factorepsr   pp_datag_datamax_norm	grad_normclipped_grad	new_gradsr   r   r   adaptive_clip_grad   s   

r&   )r   )r   r   r   )__doc__r   r   r&   r   r   r   r   <module>   s    
	