o
    ॵi                     @   s   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ G dd	 d	Zejejd
G dd de
Zejejd
G dd deZdS )    N)	clip_grad)Hooks)
OutputKeys)HOOKS)Hook)Priorityc                   @   s0   e Zd Zdd Zdd Zdd Zedd Zd	S )
OptimizerProcessorc                 C   s   |j   dS )zzInitialize the optimizer.

        This is a strategic function which can be registered by other hook's function.
        N)	optimizer	zero_gradselftrainer r   \/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/hooks/optimizer/base.pyinitialize_optimizer   s   z'OptimizerProcessor.initialize_optimizerc                 C      d S Nr   r   r   r   r   before_forward      z!OptimizerProcessor.before_forwardc                 C   sr   |D ]}|j |  |  < |j |   qt||r7|dur+| j|j fi | |j  |j	  dS dS )a  Do module backward, optimizer's step and zero_grad and clip the grads.

        This is a strategic function which can be registered by other hook's function.

        Args:
            trainer(`EpochBasedTrainer`): The trainer instance.
            loss_keys(`list`): The list of loss keys.
            cumulative_iters(`int`): The cumulative iters for gradients.
            grad_clip(`dict`): The grad clipping options.
        N)
train_outputsbackwardr   every_n_iters
clip_gradsmodel
parametersr	   stepr
   )r   r   	loss_keyscumulative_iters	grad_clipkr   r   r   r      s   
zOptimizerProcessor.backwardc                 K   s4   t tdd | } t| dkrtj| fi |S d S )Nc                 S   s   | j o| jd uS r   )requires_gradgrad)pr   r   r   <lambda>2   s    z/OptimizerProcessor.clip_grads.<locals>.<lambda>r   )listfilterlenr   clip_grad_norm_)params	clip_argsr   r   r   r   /   s   zOptimizerProcessor.clip_gradsN)__name__
__module____qualname__r   r   r   staticmethodr   r   r   r   r   r      s    r   )module_namec                   @   sL   e Zd ZdZejZddejf	dddZ	dd Z
d	d
 Zdd Zdd ZdS )OptimizerHooka  Optimizer hook

    Args:
        cumulative_iters (int): interval of gradients accumulation. Default: 1
        grad_clip (dict): Default None. Containing keys:
            max_norm (float or int): max norm of the gradients
            norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for infinity norm.
            More details please refer to `torch.nn.utils.clip_grad.clip_grad_norm_`
        loss_keys (str | list): keys list of loss
       Nreturnc                 K   s@   t |tr|g}t |ttfsJ || _|| _|| _t | _d S r   )	
isinstancestrtupler$   r   r   r   r   	processor)r   r   r   r   kwargsr   r   r   __init__F   s   
zOptimizerHook.__init__c                 C   s
   || _ d S r   )r5   )r   r5   r   r   r   set_processorS   s   
zOptimizerHook.set_processorc                 C   s   | j |_ | j| d S r   )r   r5   r   r   r   r   r   
before_runV   s   zOptimizerHook.before_runc                 C   s   | j | d S r   )r5   r   r   r   r   r   before_train_iterZ   s   zOptimizerHook.before_train_iterc                 C   s   | j || j| j| j d S r   )r5   r   r   r   r   r   r   r   r   after_train_iter]   s   zOptimizerHook.after_train_iter)r1   N)r*   r+   r,   __doc__r   ABOVE_NORMALPRIORITYr   LOSSr7   r8   r9   r:   r;   r   r   r   r   r/   7   s    
r/   c                       s.   e Zd Zd
 fdd	Zdd Zdd	 Z  ZS )NoneOptimizerHookr0   Nlossc                    s   t t| j||d || _d S )N)r   r   )superr@   r7   r   )r   r   r   r   	__class__r   r   r7   e   s   

zNoneOptimizerHook.__init__c                 C   r   r   r   r   r   r   r   r9   k   r   zNoneOptimizerHook.before_runc                 C   r   r   r   r   r   r   r   r;   n   r   z"NoneOptimizerHook.after_train_iter)r0   NrA   )r*   r+   r,   r7   r9   r;   __classcell__r   r   rC   r   r@   b   s    r@   )loggingtorch.nn.utilsr   modelscope.metainfor   modelscope.outputsr   !modelscope.trainers.hooks.builderr   modelscope.trainers.hooks.hookr   "modelscope.trainers.hooks.priorityr   r   register_moduler/   r@   r   r   r   r   <module>   s   **