o
    8wi                     @   s   d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	 d dl
Z
d dl
mZ d dlmZ d dlmZ d dlmZ d dlZd d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ G dd deeZdS )    N)	Generator)partial)AnyCallableOptionalUnion)Tensor)Module)	Optimizer)override)	Precision)	Steppable)CheckpointHooks)call)GradClipAlgorithmTypec                   @   s  e Zd ZdZdedee dee deeee ee f fddZ	e
ded	d
defddZe
dedd
dee dededdfddZe
ded	d
defddZdd
deddfddZdd
dedeg ef defddZe
dedd
deg ef dedef
ddZ		d0ded
ef dedeeeef  dee ddf
ddZd ejfdedeeef deddfd!d"Zdedeeef ddfd#d$Zdedeeef ddfd%d&Zejde d' fd(d)Z!ejde d' fd*d+Z"ejde d' fd,d-Z#ejde d' fd.d/Z$dS )1r   zBase class for all plugins handling the precision-specific parts of the training.

    The class attribute precision must be overwritten in child classes. The default value reflects fp32 training.

    model
optimizerslr_schedulersreturnc                 C   s
   |||fS )zAConnects this plugin to the accelerator and the training process. )selfr   r   r   r   r   j/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision.pyconnect(   s   
zPrecision.connecttensormodulezpl.LightningModulec                 C   s&   |j }t|d| t|d| |S )Non_before_backward)trainerr   _call_callback_hooks_call_lightning_module_hook)r   r   r   r   r   r   r   pre_backward.   s   zPrecision.pre_backward	optimizerargskwargsNc                 O   s   |j |g|R i | dS )a  Performs the actual backpropagation.

        Args:
            tensor: the loss value obtained from the closure
            model: the model to be optimized
            optimizer: current optimizer being used. ``None`` if using manual optimization
            \*args: Positional arguments intended for the actual function that performs the backward, like
                :meth:`~torch.Tensor.backward`.
            \**kwargs: Keyword arguments for the same purpose as ``*args``.

        N)backward)r   r   r   r    r!   r"   r   r   r   r#   5   s   zPrecision.backwardc                 C   s*   |  }|j}t|d t|d |S )Non_after_backward)detachr   r   r   r   )r   r   r   closure_lossr   r   r   r   post_backwardK   s
   zPrecision.post_backwardc                 C   s<   |j }t|d| t|d| | j|||j|jd dS )z:Utility to share some code after the closure has been run.on_before_optimizer_step)gradient_clip_algorithmN)r   r   r   r   _clip_gradientsgradient_clip_valr)   )r   r   r    r   r   r   r   _after_closureT   s   
zPrecision._after_closureclosurec                 C   s   | }|  || |S )an  This double-closure allows makes sure the ``closure`` is executed before the ``on_before_optimizer_step``
        hook is called.

        The closure (generally) runs ``backward`` so this allows inspecting gradients in this hook. This structure is
        consistent with the ``Precision`` subclasses that cannot pass ``optimizer.step(closure)`` directly.

        )r,   )r   r   r    r-   closure_resultr   r   r   _wrap_closure`   s   zPrecision._wrap_closurec                 K   s$   t | j|||}|jdd|i|S )zHook to run the optimizer step.r-   Nr   )r   r/   step)r   r    r   r-   r"   r   r   r   optimizer_stepq   s   	zPrecision.optimizer_stepclip_valr)   c                 C   s0   t |tjr	|jsd S tj|jd|||d d S )Nconfigure_gradient_clipping)r+   r)   )
isinstanceplLightningModuleautomatic_optimizationr   r   r   )r   r   r    r2   r)   r   r   r   r*   }   s   
zPrecision._clip_gradientsg        c                 C   sD   |dkrdS |t jkr| || dS |t jkr | || dS dS )zClips the gradients.r   N)r   VALUEclip_grad_by_valueNORMclip_grad_by_norm)r   r    r2   r)   r   r   r   clip_gradients   s   

zPrecision.clip_gradientsc                 C   s    |  |}tjjj||d dS )zClip gradients by value.)
clip_valueN)main_paramstorchnnutilsclip_grad_value_r   r    r2   
parametersr   r   r   r9      s   
zPrecision.clip_grad_by_valuec                 C   s   |  |}tjj|| dS )zClip gradients by norm.N)r>   r?   r@   rA   clip_grad_norm_rC   r   r   r   r;      s   
zPrecision.clip_grad_by_norm)NNNc                 c   6    |    dV  W d   dS 1 sw   Y  dS )z'A contextmanager for the training step.Nforward_contextr   r   r   r   train_step_context      
"zPrecision.train_step_contextc                 c   rF   )z)A contextmanager for the validation step.NrG   rI   r   r   r   val_step_context   rK   zPrecision.val_step_contextc                 c   rF   )z#A contextmanager for the test step.NrG   rI   r   r   r   test_step_context   rK   zPrecision.test_step_contextc                 c   rF   )z&A contextmanager for the predict step.NrG   rI   r   r   r   predict_step_context   rK   zPrecision.predict_step_context)NN)%__name__
__module____qualname____doc__r	   listr
   r   tupler   r   r   r   r   r   r#   r'   r,   r   r/   r1   r   intfloatr   r*   r:   r<   r9   r;   
contextlibcontextmanagerr   rJ   rL   rM   rN   r   r   r   r   r   !   s    







r   )rW   collections.abcr   	functoolsr   typingr   r   r   r   r?   r   torch.nnr	   torch.optimr
   typing_extensionsr   pytorch_lightningr5   lightning_fabric.pluginsr   FabricPrecision lightning_fabric.utilities.typesr   pytorch_lightning.core.hooksr   pytorch_lightning.trainerr   pytorch_lightning.utilitiesr   r   r   r   r   <module>   s    