o
    oi                     @   s   d dl Z d dlmZ d dlmZmZmZmZmZm	Z	m
Z
 d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlm Z  G dd deeZdS )    N)partial)AnyCallable	GeneratorListOptionalTupleUnion)Tensor)Module)	Optimizer)override)	Precision)	Steppable)CheckpointHooks)call)GradClipAlgorithmTypec                   @   s  e Zd ZdZdedee dee deeee ee f fddZ	e
ded	d
defddZe
dedd
dee dededdfddZe
ded	d
defddZdd
deddfddZdd
dedeg ef defddZe
dedd
deg ef dedef
ddZ		d0ded
ef dedeeeef  dee ddf
ddZd ejfdedeeef deddfd!d"Zdedeeef ddfd#d$Zdedeeef ddfd%d&Zejde d' fd(d)Z!ejde d' fd*d+Z"ejde d' fd,d-Z#ejde d' fd.d/Z$dS )1r   zBase class for all plugins handling the precision-specific parts of the training.

    The class attribute precision must be overwritten in child classes. The default value reflects fp32 training.

    model
optimizerslr_schedulersreturnc                 C   s
   |||fS )zAConnects this plugin to the accelerator and the training process. )selfr   r   r   r   r   a/home/ubuntu/.local/lib/python3.10/site-packages/lightning/pytorch/plugins/precision/precision.pyconnect'   s   
zPrecision.connecttensormodulezpl.LightningModulec                 C   s&   |j }t|d| t|d| |S )Non_before_backward)trainerr   _call_callback_hooks_call_lightning_module_hook)r   r   r   r   r   r   r   pre_backward-   s   zPrecision.pre_backward	optimizerargskwargsNc                 O   s   |j |g|R i | dS )a  Performs the actual backpropagation.

        Args:
            tensor: the loss value obtained from the closure
            model: the model to be optimized
            optimizer: current optimizer being used. ``None`` if using manual optimization
            \*args: Positional arguments intended for the actual function that performs the backward, like
                :meth:`~torch.Tensor.backward`.
            \**kwargs: Keyword arguments for the same purpose as ``*args``.

        N)backward)r   r   r   r"   r#   r$   r   r   r   r%   4   s   zPrecision.backwardc                 C   s*   |  }|j}t|d t|d |S )Non_after_backward)detachr   r   r   r    )r   r   r   closure_lossr   r   r   r   post_backwardJ   s
   zPrecision.post_backwardc                 C   s<   |j }t|d| t|d| | j|||j|jd dS )z:Utility to share some code after the closure has been run.on_before_optimizer_step)gradient_clip_algorithmN)r   r   r   r    _clip_gradientsgradient_clip_valr+   )r   r   r"   r   r   r   r   _after_closureS   s   
zPrecision._after_closureclosurec                 C   s   | }|  || |S )an  This double-closure allows makes sure the ``closure`` is executed before the ``on_before_optimizer_step``
        hook is called.

        The closure (generally) runs ``backward`` so this allows inspecting gradients in this hook. This structure is
        consistent with the ``Precision`` subclasses that cannot pass ``optimizer.step(closure)`` directly.

        )r.   )r   r   r"   r/   closure_resultr   r   r   _wrap_closure_   s   zPrecision._wrap_closurec                 K   s$   t | j|||}|jdd|i|S )zHook to run the optimizer step.r/   Nr   )r   r1   step)r   r"   r   r/   r$   r   r   r   optimizer_stepp   s   	zPrecision.optimizer_stepclip_valr+   c                 C   s0   t |tjr	|jsd S tj|jd|||d d S )Nconfigure_gradient_clipping)r-   r+   )
isinstanceplLightningModuleautomatic_optimizationr   r    r   )r   r   r"   r4   r+   r   r   r   r,   |   s   
zPrecision._clip_gradientsg        c                 C   sD   |dkrdS |t jkr| || dS |t jkr | || dS dS )zClips the gradients.r   N)r   VALUEclip_grad_by_valueNORMclip_grad_by_norm)r   r"   r4   r+   r   r   r   clip_gradients   s   

zPrecision.clip_gradientsc                 C   s    |  |}tjjj||d dS )zClip gradients by value.)
clip_valueN)main_paramstorchnnutilsclip_grad_value_r   r"   r4   
parametersr   r   r   r;      s   
zPrecision.clip_grad_by_valuec                 C   s   |  |}tjj|| dS )zClip gradients by norm.N)r@   rA   rB   rC   clip_grad_norm_rE   r   r   r   r=      s   
zPrecision.clip_grad_by_norm)NNNc                 c   6    |    dV  W d   dS 1 sw   Y  dS )z'A contextmanager for the training step.Nforward_contextr   r   r   r   train_step_context      
"zPrecision.train_step_contextc                 c   rH   )z)A contextmanager for the validation step.NrI   rK   r   r   r   val_step_context   rM   zPrecision.val_step_contextc                 c   rH   )z#A contextmanager for the test step.NrI   rK   r   r   r   test_step_context   rM   zPrecision.test_step_contextc                 c   rH   )z&A contextmanager for the predict step.NrI   rK   r   r   r   predict_step_context   rM   zPrecision.predict_step_context)NN)%__name__
__module____qualname____doc__r   r   r   r   r   r   r   r
   r!   r   r   r%   r)   r.   r   r1   r3   r	   intfloatr   r,   r<   r>   r;   r=   
contextlibcontextmanagerr   rL   rN   rO   rP   r   r   r   r   r       s    







r   )!rW   	functoolsr   typingr   r   r   r   r   r   r	   rA   r
   torch.nnr   torch.optimr   typing_extensionsr   lightning.pytorchpytorchr7   lightning.fabric.pluginsr   FabricPrecision lightning.fabric.utilities.typesr   lightning.pytorch.core.hooksr   lightning.pytorch.trainerr   lightning.pytorch.utilitiesr   r   r   r   r   <module>   s   $