o
    oi                     @   s   d dl mZmZmZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ G dd deZdedefddZdS )    )AnyContextManagerDictLiteralOptionalN)apply_to_collection)Tensor)Module)LBFGS	Optimizer)override)	Precision)_convert_fp_tensor)_TORCH_GREATER_EQUAL_2_4)Optimizablec                       s  e Zd ZdZ	d"ded deded ddfd	d
Zede	fddZ
ededefddZededefddZededee dededdf
 fddZedededef fddZedeeef fddZedeeef ddfddZededdfd d!Z  ZS )#MixedPrecisionaF  Plugin for Automatic Mixed Precision (AMP) training with ``torch.autocast``.

    Args:
        precision: Whether to use ``torch.float16`` (``'16-mixed'``) or ``torch.bfloat16`` (``'bf16-mixed'``).
        device: The device for ``torch.autocast``.
        scaler: An optional :class:`torch.cuda.amp.GradScaler` to use.

    N	precision16-mixed
bf16-mixeddevicescalerztorch.amp.GradScalerreturnc                 C   s   |dvrt dt| j d|d|| _|d u r-| jdkr-tr'tjj|dntjj }|d ur>| jdkr>t d| d	|| _	|| _
| jdkrOtj| _d S tj| _d S )
Nr   zPassed `z(precision=z1)`. Precision must be '16-mixed' or 'bf16-mixed'.r   )r   r   z6`precision='bf16-mixed'` does not use a scaler, found .)
ValueErrortype__name__r   r   torchamp
GradScalercudar   r   bfloat16float16_desired_input_dtype)selfr   r   r    r%   Z/home/ubuntu/.local/lib/python3.10/site-packages/lightning/fabric/plugins/precision/amp.py__init__'   s   "zMixedPrecision.__init__c                 C   s   t j| j| jdS )N)dtype)r   autocastr   r#   r$   r%   r%   r&   forward_context=      zMixedPrecision.forward_contextdatac                 C   s   t |tt| jdS N)functionr(   dst_type)r   r   r   r#   r$   r-   r%   r%   r&   convert_inputA   r,   zMixedPrecision.convert_inputc                 C   s   t |ttt dS r.   )r   r   r   r   get_default_dtyper1   r%   r%   r&   convert_outputE   s   zMixedPrecision.convert_outputtensormodelargskwargsc                    s6   | j d ur| j |}t j||g|R i | d S N)r   scalesuperbackward)r$   r5   r6   r7   r8   	__class__r%   r&   r<   I   s   
 zMixedPrecision.backward	optimizerc                    sR   | j d u rt j|fi |S t|trtd| j j|fi |}| j   |S )Nz/AMP and the LBFGS optimizer are not compatible.)r   r;   optimizer_step
isinstancer
   	TypeErrorstepupdate)r$   r?   r8   step_outputr=   r%   r&   r@   O   s   


zMixedPrecision.optimizer_stepc                 C   s   | j d ur
| j  S i S r9   )r   
state_dictr*   r%   r%   r&   rF   _   s   

zMixedPrecision.state_dictrF   c                 C   s   | j d ur| j | d S d S r9   )r   load_state_dict)r$   rF   r%   r%   r&   rG   e   s   
zMixedPrecision.load_state_dictc                 C   s0   | j }|d urt|rtd|| d S d S )NzKGradient clipping is not implemented for optimizers handling the unscaling.)r   _optimizer_handles_unscalingNotImplementedErrorunscale_)r$   r?   r   r%   r%   r&   unscale_gradientsj   s   z MixedPrecision.unscale_gradientsr9   )r   
__module____qualname____doc__r   strr   r'   r   r   r+   r   r2   r4   r   r	   r<   r   r@   r   rF   rG   r   rK   __classcell__r%   r%   r=   r&   r      sD    
(r   r?   r   c                 C   s   t | ddS )aT  Determines whether a PyTorch optimizer handles unscaling gradients in the step method rather than through the
    :class:`torch.cuda.amp.GradScaler`.

    Since, the current implementation of this function checks a PyTorch internal variable on the optimizer, the return
    value will only be reliable for built-in PyTorch optimizers.

    _step_supports_amp_scalingF)getattr)r?   r%   r%   r&   rH   s   s   rH   )typingr   r   r   r   r   r   #lightning_utilities.core.apply_funcr   r   torch.nnr	   torch.optimr
   r   typing_extensionsr   ,lightning.fabric.plugins.precision.precisionr   (lightning.fabric.plugins.precision.utilsr   "lightning.fabric.utilities.importsr    lightning.fabric.utilities.typesr   r   boolrH   r%   r%   r%   r&   <module>   s   V