o
    oi                     @   s   d dl mZ d dlmZmZmZmZ d dlZd dlm	Z	 d dlm
Z
 d dlmZ d dlmZmZ d dlmZ d d	lmZmZ d d
lmZ erLd dlmZ ed ZG dd deZdS )    )nullcontext)TYPE_CHECKINGAnyContextManagerLiteralN)apply_to_collection)Tensor)Module)get_argsoverride)	Precision)_convert_fp_tensor_DtypeContextManager)	Steppable)DeepSpeedEngine)32-true16-true	bf16-true16-mixed
bf16-mixedc                   @   s   e Zd ZdZdeddfddZededefdd	Zede	fd
dZ
ede	fddZededefddZededefddZededddededdf
ddZedededefddZdS )DeepSpeedPrecisiona  Precision plugin for DeepSpeed integration.

    Args:
        precision: Full precision (32-true), half precision (16-true, bf16-true) or
            mixed precision (16-mixed, bf16-mixed).

    Raises:
        ValueError:
            If unsupported ``precision`` is provided.

    	precisionreturnNc                 C   sV   t t}||vrtd|d| d|| _tjtjtjtjtjd}|| j | _d S )Nz`precision=z>)` is not supported in DeepSpeed. `precision` must be one of: .)r   r   r   r   r   )	r
   _PRECISION_INPUT
ValueErrorr   torchbfloat16float16float32_desired_dtype)selfr   supported_precisionprecision_to_type r$   `/home/ubuntu/.local/lib/python3.10/site-packages/lightning/fabric/plugins/precision/deepspeed.py__init__.   s   zDeepSpeedPrecision.__init__modulec                 C   s   d| j v r|j| jdS |S )Ntrue)dtype)r   tor    )r!   r'   r$   r$   r%   convert_module@   s   
z!DeepSpeedPrecision.convert_modulec                 C   s   d| j vrt S t| jS )Nr(   )r   r   r   r    r!   r$   r$   r%   tensor_init_contextF   s   

z&DeepSpeedPrecision.tensor_init_contextc                 C   s   |   S )N)r-   r,   r$   r$   r%   module_init_contextL   s   z&DeepSpeedPrecision.module_init_contextdatac                 C   s   t |tt| jdS N)functionr)   dst_type)r   r   r   r    r!   r/   r$   r$   r%   convert_inputP   s   z DeepSpeedPrecision.convert_inputc                 C   s   t |ttt dS r0   )r   r   r   r   get_default_dtyper3   r$   r$   r%   convert_outputT   s   z!DeepSpeedPrecision.convert_outputtensormodelr   argskwargsc                 O   s   |j |g|R i | dS )z3Performs back-propagation using DeepSpeed's engine.N)backward)r!   r7   r8   r9   r:   r$   r$   r%   r;   X   s   zDeepSpeedPrecision.backward	optimizerc                 K   s   |j di |S )Nr$   )step)r!   r<   r:   r$   r$   r%   optimizer_step]   s   z!DeepSpeedPrecision.optimizer_step)__name__
__module____qualname____doc__r   r&   r   r	   r+   r   r-   r.   r   r4   r6   r   r;   r   r>   r$   r$   r$   r%   r   !   s.     r   )
contextlibr   typingr   r   r   r   r   #lightning_utilities.core.apply_funcr   r   torch.nnr	   typing_extensionsr
   r   ,lightning.fabric.plugins.precision.precisionr   (lightning.fabric.plugins.precision.utilsr   r    lightning.fabric.utilities.typesr   	deepspeedr   r   r   r$   r$   r$   r%   <module>   s   