o
    zi                     @   sl   d Z ddlmZmZ ddlmZ ddlZddlm	Z	 ddl
mZ ddlmZ ddlmZ G d	d
 d
e	ZdS )z
Gradient Accumulator
====================

Change gradient accumulation factor according to scheduling.
Trainer also calls ``optimizer.step()`` for the last indivisible step number.

    )AnyDict)overrideN)Callback)MisconfigurationException)is_overridden)rank_zero_warnc                       sv   e Zd ZdZdeeef f fddZdefddZdedefd	d
Z	e
dddZe
dddeddfddZ  ZS )GradientAccumulationSchedulera  Change gradient accumulation factor according to scheduling.

    Args:
        scheduling: scheduling in format {epoch: accumulation_factor}

    Note:
        The argument scheduling is a dictionary. Each key represent an epoch and
        its associated accumulation factor value.
        Warning: Epoch are zero-indexed c.f it means if you want to change
        the accumulation factor after 4 epochs, set ``Trainer(accumulate_grad_batches={4: factor})``
        or ``GradientAccumulationScheduler(scheduling={4: factor})``.
        For more info check the example below.

    Raises:
        TypeError:
            If ``scheduling`` is an empty ``dict``,
            or not all keys and values of ``scheduling`` are integers.
        IndexError:
            If ``minimal_epoch`` is less than 0.

    Example::

        >>> from pytorch_lightning import Trainer
        >>> from pytorch_lightning.callbacks import GradientAccumulationScheduler

        # from epoch 5, it starts accumulating every 2 batches. Here we have 4 instead of 5
        # because epoch (key) should be zero-indexed.
        >>> accumulator = GradientAccumulationScheduler(scheduling={4: 2})
        >>> trainer = Trainer(callbacks=[accumulator])

    
schedulingc                    s   t    |stdtdd |D r tdt|  dtdd | D r7tdt|  dt| }|dk rIt	d	| d
|dkrT|
ddi || _t| | _d S )Nz(Empty dict cannot be interpreted correctc                 s   $    | ]}t |t p|d k V  qdS )r   N
isinstanceint).0key r   o/home/ubuntu/.local/lib/python3.10/site-packages/pytorch_lightning/callbacks/gradient_accumulation_scheduler.py	<genexpr>I      " z9GradientAccumulationScheduler.__init__.<locals>.<genexpr>z7Epoch should be an int greater than or equal to 0. Got .c                 s   r      Nr   )r   valuer   r   r   r   N   r   z9Accumulation factor should be an int greater than 0. Got r   zEpochs indexing from 1, epoch z cannot be interpreted correctr   )super__init__	TypeErroranyr   listkeysvaluesmin
IndexErrorupdater
   sortedepochs)selfr
   minimal_epoch	__class__r   r   r   C   s$   
z&GradientAccumulationScheduler.__init__returnc                 C   s   t dd | j D S )Nc                 s   s    | ]}|d kV  qdS r   r   )r   vr   r   r   r   ]   s    zQGradientAccumulationScheduler.going_to_accumulate_grad_batches.<locals>.<genexpr>)r   r
   r   )r%   r   r   r    going_to_accumulate_grad_batches\   s   z>GradientAccumulationScheduler.going_to_accumulate_grad_batchesepochc                 C   s0   d}t | jD ]}||kr| j| } |S q|S )Nr   )reversedr$   r
   )r%   r,   accumulate_grad_batches
iter_epochr   r   r   get_accumulate_grad_batches_   s   
z9GradientAccumulationScheduler.get_accumulate_grad_batchestrainer
pl.Trainer	pl_modulepl.LightningModuleNc                 C   s   |j stdtd|}td|}|  }|p|}|r!|r!td ddlm} t|j|r9tdt	|jj
 d|jd	krBtd
dS )zhPerformns a configuration validation before training starts and raises errors for incompatible settings.zAutomatic gradient accumulation and the `GradientAccumulationScheduler` is not supported for
                manual optimization. Please remove the callback or switch to automatic optimization.optimizer_stepoptimizer_zero_gradzWhen using `Trainer(accumulate_grad_batches != 1)` and overriding `LightningModule.optimizer_{step,zero_grad}`, the hooks will not be called on every batch (rather, they are called on every optimization step).r   )DeepSpeedStrategyzThe `zE` does not support `accumulate_grad_batches` changing between epochs.r   zYou have set `accumulate_grad_batches` and are using the `GradientAccumulationScheduler` callback. Either remove `accumulate_grad_batches` from the Trainer or remove the callback.N)automatic_optimizationRuntimeErrorr   r+   r   pytorch_lightning.strategiesr7   r   strategytype__name__r.   
ValueError)r%   r1   r3   overridden_optimizer_stepoverridden_optimizer_zero_gradr+   %has_overridden_optimization_functionsr7   r   r   r   on_train_startg   s,   


z,GradientAccumulationScheduler.on_train_start_c                 G   s   |  |j|_d S )N)r0   current_epochr.   )r%   r1   rC   r   r   r   on_train_epoch_start   s   z2GradientAccumulationScheduler.on_train_epoch_start)r1   r2   r3   r4   r)   N)r=   
__module____qualname____doc__r   r   r   boolr+   r0   r   rB   r   rE   __classcell__r   r   r'   r   r	   "   s     " r	   )rH   typingr   r   typing_extensionsr   pytorch_lightningpl$pytorch_lightning.callbacks.callbackr   &pytorch_lightning.utilities.exceptionsr   )pytorch_lightning.utilities.model_helpersr   %pytorch_lightning.utilities.rank_zeror   r	   r   r   r   r   <module>   s   	