o
    8wi                     @   s   d dl mZ d dlmZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ G dd deZdedefddZdS )    )AbstractContextManager)AnyLiteralOptionalN)apply_to_collection)Tensor)Module)LBFGS	Optimizer)override)	Precision)_convert_fp_tensor)_TORCH_GREATER_EQUAL_2_4)Optimizablec                       s  e Zd ZdZ	d"ded deded ddfd	d
Zede	fddZ
ededefddZededefddZededee dededdf
 fddZedededef fddZedeeef fddZedeeef ddfddZededdfd d!Z  ZS )#MixedPrecisionaF  Plugin for Automatic Mixed Precision (AMP) training with ``torch.autocast``.

    Args:
        precision: Whether to use ``torch.float16`` (``'16-mixed'``) or ``torch.bfloat16`` (``'bf16-mixed'``).
        device: The device for ``torch.autocast``.
        scaler: An optional :class:`torch.cuda.amp.GradScaler` to use.

    N	precision16-mixed
bf16-mixeddevicescalerztorch.amp.GradScalerreturnc                 C   s   |dvrt dt| j d|d|| _|d u r-| jdkr-tr'tjj|dntjj }|d ur>| jdkr>t d| d	|| _	|| _
| jdkrOtj| _d S tj| _d S )
Nr   zPassed `z(precision=z1)`. Precision must be '16-mixed' or 'bf16-mixed'.r   )r   r   z6`precision='bf16-mixed'` does not use a scaler, found .)
ValueErrortype__name__r   r   torchamp
GradScalercudar   r   bfloat16float16_desired_input_dtype)selfr   r   r    r$   c/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/lightning_fabric/plugins/precision/amp.py__init__(   s   "zMixedPrecision.__init__c                 C   s   t j| j| jdS )N)dtype)r   autocastr   r"   r#   r$   r$   r%   forward_context>      zMixedPrecision.forward_contextdatac                 C   s   t |tt| jdS N)functionr'   dst_type)r   r   r   r"   r#   r,   r$   r$   r%   convert_inputB   r+   zMixedPrecision.convert_inputc                 C   s   t |ttt dS r-   )r   r   r   r   get_default_dtyper0   r$   r$   r%   convert_outputF   s   zMixedPrecision.convert_outputtensormodelargskwargsc                    s6   | j d ur| j |}t j||g|R i | d S N)r   scalesuperbackward)r#   r4   r5   r6   r7   	__class__r$   r%   r;   J   s   
 zMixedPrecision.backward	optimizerc                    sR   | j d u rt j|fi |S t|trtd| j j|fi |}| j   |S )Nz/AMP and the LBFGS optimizer are not compatible.)r   r:   optimizer_step
isinstancer	   	TypeErrorstepupdate)r#   r>   r7   step_outputr<   r$   r%   r?   P   s   


zMixedPrecision.optimizer_stepc                 C   s   | j d ur
| j  S i S r8   )r   
state_dictr)   r$   r$   r%   rE   `   s   

zMixedPrecision.state_dictrE   c                 C   s   | j d ur| j | d S d S r8   )r   load_state_dict)r#   rE   r$   r$   r%   rF   f   s   
zMixedPrecision.load_state_dictc                 C   s0   | j }|d urt|rtd|| d S d S )NzKGradient clipping is not implemented for optimizers handling the unscaling.)r   _optimizer_handles_unscalingNotImplementedErrorunscale_)r#   r>   r   r$   r$   r%   unscale_gradientsk   s   z MixedPrecision.unscale_gradientsr8   )r   
__module____qualname____doc__r   strr   r&   r   r   r*   r   r1   r3   r   r   r;   r   r?   dictrE   rF   r
   rJ   __classcell__r$   r$   r<   r%   r      sD    
(r   r>   r   c                 C   s   t | ddS )aT  Determines whether a PyTorch optimizer handles unscaling gradients in the step method rather than through the
    :class:`torch.cuda.amp.GradScaler`.

    Since, the current implementation of this function checks a PyTorch internal variable on the optimizer, the return
    value will only be reliable for built-in PyTorch optimizers.

    _step_supports_amp_scalingF)getattr)r>   r$   r$   r%   rG   t   s   rG   )
contextlibr   typingr   r   r   r   #lightning_utilities.core.apply_funcr   r   torch.nnr   torch.optimr	   r
   typing_extensionsr   ,lightning_fabric.plugins.precision.precisionr   (lightning_fabric.plugins.precision.utilsr   "lightning_fabric.utilities.importsr    lightning_fabric.utilities.typesr   r   boolrG   r$   r$   r$   r%   <module>   s   V