o
    zip                     @   s   d dl mZmZmZmZmZmZ d dlZd dlm	Z	 d dlm
Z
 d dlmZmZ d dlZd dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZ er`d dlmZ d dl m!Z! G dd deZ"dS )    )TYPE_CHECKINGAnyCallableContextManagerDictOptionalN)apply_to_collection)Tensor)get_argsoverride)_optimizer_handles_unscaling)_PRECISION_INPUT)_convert_fp_tensor_DtypeContextManager)Optimizable)	Precision)MisconfigurationExceptionMixedPrecisionShardedGradScalerc                       s>  e Zd ZdZd+deded ddfddZed	ed
eddfddZ	e
d,ddZedefddZedefddZedefddZededefddZededefddZededddef fddZed ed!dd"eg ef d#edef
 fd$d%Zedeeef fd&d'Zed(eeef ddfd)d*Z  ZS )-FSDPPrecisiona  Precision plugin for training with Fully Sharded Data Parallel (FSDP).

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Args:
        precision: Full precision (32-true), half precision (16-true, bf16-true) or
            mixed precision (16-mixed, bf16-mixed).
        scaler: An optional :class:`torch.distributed.fsdp.sharded_grad_scaler.ShardedGradScaler` to use.

    Raises:
        ValueError:
            If unsupported ``precision`` is provided.

    N	precisionscalerr   returnc                 C   s   t t}||vrtd|d| dddlm} |d ur-| jdkr-td|d| d|d u r8|dkr8| nd | _|| _tjtjtj	tj
tjd}|| j | _d S )	Nz`precision=z9)` is not supported in FSDP. `precision` must be one of: .r   r   16-mixedz` does not use a scaler, found )
bf16-mixedr   	bf16-true16-true32-true)r
   r   
ValueError*torch.distributed.fsdp.sharded_grad_scalerr   r   r   torchfloat32bfloat16float16_desired_input_dtype)selfr   r   supported_precisionr   precision_to_type r+   \/home/ubuntu/.local/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/fsdp.py__init__2   s&   zFSDPPrecision.__init_____c                 O   s   t d| jj d)NzA`gradient_clip_algorithm='norm'` is currently not supported for ``)r   	__class____name__)r(   r.   r/   r+   r+   r,   clip_grad_by_normK   s   zFSDPPrecision.clip_grad_by_normTorchMixedPrecisionc                 C   s   ddl m} | jdkrtj}tj }}n?| jdkr"tj}tj }}n1| jdkr/tj } }}n$| jdkr<tj } }}n| jdkrJtj}tj }}n	td| jd	||||d
S )Nr   r   r   r   r   r   r    z-Was unable to infer precision type, received r   )param_dtypereduce_dtypebuffer_dtype)2torch.distributed.fsdp.fully_sharded_data_parallelr   r   r#   r$   r&   r%   r   )r(   r4   r5   r6   r7   r+   r+   r,   mixed_precision_configU   s(   




z$FSDPPrecision.mixed_precision_configc                 C   s
   t | jS N)r   r'   r(   r+   r+   r,   tensor_init_contexto   s   
z!FSDPPrecision.tensor_init_contextc                 C   s   t | jjptjS r:   )r   r9   r5   r#   r$   r;   r+   r+   r,   module_init_contexts      z!FSDPPrecision.module_init_contextc                 C   s8   d| j v rtjd| j dkrtjdS tjdS t| jS )Nmixedcudar   )dtype)r   r#   autocastr%   r&   r   r'   r;   r+   r+   r,   forward_contextw   s   
$
zFSDPPrecision.forward_contextdatac                 C   s   t |tt| jdS N)functionrA   dst_type)r   r   r	   r'   r(   rD   r+   r+   r,   convert_input}   r>   zFSDPPrecision.convert_inputc                 C   s   t |ttt dS rE   )r   r   r	   r#   get_default_dtyperH   r+   r+   r,   convert_output   s   zFSDPPrecision.convert_outputtensormodulezpl.LightningModulec                    s$   | j d ur| j |}t ||S r:   )r   scalesuperpre_backward)r(   rL   rM   r1   r+   r,   rP      s   
zFSDPPrecision.pre_backward	optimizermodelclosurekwargsc                    s   | j d u rt j|f||d|S | }t|s| j | | || |d u }|jr.|s?| j j|fi |}| j   |S |S )N)rS   rT   )	r   rO   optimizer_stepr   unscale__after_closureautomatic_optimizationstepupdate)r(   rR   rS   rT   rU   closure_resultskipped_backwardstep_outputrQ   r+   r,   rV      s   


zFSDPPrecision.optimizer_stepc                 C   s   | j d ur
| j  S i S r:   )r   
state_dictr;   r+   r+   r,   r_      s   

zFSDPPrecision.state_dictr_   c                 C   s   | j d ur| j | d S d S r:   )r   load_state_dict)r(   r_   r+   r+   r,   r`      s   
zFSDPPrecision.load_state_dictr:   )r   r4   )r2   
__module____qualname____doc__r   r   r-   r   r   r3   propertyr9   r   r<   r=   rC   rI   rK   r	   rP   r   r   rV   r   strr_   r`   __classcell__r+   r+   rQ   r,   r   "   sF    	
$r   )#typingr   r   r   r   r   r   r#   lightning_utilitiesr   r	   typing_extensionsr
   r   pytorch_lightningpl&lightning_fabric.plugins.precision.ampr   'lightning_fabric.plugins.precision.fsdpr   (lightning_fabric.plugins.precision.utilsr   r    lightning_fabric.utilities.typesr   -pytorch_lightning.plugins.precision.precisionr   &pytorch_lightning.utilities.exceptionsr   r8   r   r4   r"   r   r   r+   r+   r+   r,   <module>   s     