o
    8wi|                     @   s   d dl mZ d dlmZmZmZmZ d dlZd dlm	Z	 d dlm
Z
 d dlmZ d dlmZmZ d dlZd dlmZ d d	lmZ d d
lmZmZ d dlmZ d dlmZ d dlmZ erhd dlm Z! d dl"m#Z# G dd deZ$dS )    )AbstractContextManager)TYPE_CHECKINGAnyCallableOptionalN)apply_to_collection)Tensor)Module)get_argsoverride)_optimizer_handles_unscaling)_PRECISION_INPUT)_convert_fp_tensor_DtypeContextManager)Optimizable)	Precision)MisconfigurationExceptionMixedPrecisionShardedGradScalerc                       sT  e Zd ZdZd-deded ddfddZed	edefd
dZ	ede
de
ddfddZed.ddZedefddZedefddZedefddZede
de
fddZede
de
fddZeded	ddef fd d!Zed"ed#dd$eg e
f d%e
de
f
 fd&d'Zedeee
f fd(d)Zed*eee
f ddfd+d,Z  ZS )/FSDPPrecisiona  Precision plugin for training with Fully Sharded Data Parallel (FSDP).

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Args:
        precision: Full precision (32-true), half precision (16-true, bf16-true) or
            mixed precision (16-mixed, bf16-mixed).
        scaler: An optional :class:`torch.distributed.fsdp.sharded_grad_scaler.ShardedGradScaler` to use.

    Raises:
        ValueError:
            If unsupported ``precision`` is provided.

    N	precisionscalerr   returnc                 C   s   t t}||vrtd|d| dddlm} |d ur-| jdkr-td|d| d|d u r8|dkr8| nd | _|| _tjtjtj	tj
tjd}|| j | _d S )	Nz`precision=z9)` is not supported in FSDP. `precision` must be one of: .r   r   16-mixedz` does not use a scaler, found )
bf16-mixedr   	bf16-true16-true32-true)r
   r   
ValueError*torch.distributed.fsdp.sharded_grad_scalerr   r   r   torchfloat32bfloat16float16_desired_input_dtype)selfr   r   supported_precisionr   precision_to_type r+   e/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/fsdp.py__init__4   s&   zFSDPPrecision.__init__modulec                 C   s   d| j v r|j| jdS |S )Ntruedtype)r   tor'   )r(   r.   r+   r+   r,   convert_moduleM   s   
zFSDPPrecision.convert_module___c                 O   s   t d| jj d)NzA`gradient_clip_algorithm='norm'` is currently not supported for ``)r   	__class____name__)r(   r4   r5   r+   r+   r,   clip_grad_by_normS   s   zFSDPPrecision.clip_grad_by_normTorchMixedPrecisionc                 C   s   ddl m} | jdkrtj}tj }}n?| jdkr"tj}tj }}n1| jdkr/tj } }}n$| jdkr<tj } }}n| jdkrJtj}tj }}n	td| jd	||||d
S )Nr   r   r   r   r   r   r    z-Was unable to infer precision type, received r   )param_dtypereduce_dtypebuffer_dtype)2torch.distributed.fsdp.fully_sharded_data_parallelr   r   r#   r$   r&   r%   r   )r(   r:   r;   r<   r=   r+   r+   r,   mixed_precision_config]   s(   




z$FSDPPrecision.mixed_precision_configc                 C   s
   t | jS N)r   r'   r(   r+   r+   r,   tensor_init_contextw   s   
z!FSDPPrecision.tensor_init_contextc                 C   s   t | jjptjS r@   )r   r?   r;   r#   r$   rA   r+   r+   r,   module_init_context{      z!FSDPPrecision.module_init_contextc                 C   s8   d| j v rtjd| j dkrtjdS tjdS t| jS )Nmixedcudar   r0   )r   r#   autocastr%   r&   r   r'   rA   r+   r+   r,   forward_context   s   
$
zFSDPPrecision.forward_contextdatac                 C   s   t |tt| jdS N)functionr1   dst_type)r   r   r   r'   r(   rI   r+   r+   r,   convert_input   rD   zFSDPPrecision.convert_inputc                 C   s   t |ttt dS rJ   )r   r   r   r#   get_default_dtyperM   r+   r+   r,   convert_output   s   zFSDPPrecision.convert_outputtensorzpl.LightningModulec                    s$   | j d ur| j |}t ||S r@   )r   scalesuperpre_backward)r(   rQ   r.   r7   r+   r,   rT      s   
zFSDPPrecision.pre_backward	optimizermodelclosurekwargsc                    s   | j d u rt j|f||d|S | }t|s| j | | || |d u }|jr.|s?| j j|fi |}| j   |S |S )N)rW   rX   )	r   rS   optimizer_stepr   unscale__after_closureautomatic_optimizationstepupdate)r(   rV   rW   rX   rY   closure_resultskipped_backwardstep_outputrU   r+   r,   rZ      s   


zFSDPPrecision.optimizer_stepc                 C   s   | j d ur
| j  S i S r@   )r   
state_dictrA   r+   r+   r,   rc      s   

zFSDPPrecision.state_dictrc   c                 C   s   | j d ur| j | d S d S r@   )r   load_state_dict)r(   rc   r+   r+   r,   rd      s   
zFSDPPrecision.load_state_dictr@   )r   r:   )r8   
__module____qualname____doc__r   r   r-   r   r	   r3   r   r9   propertyr?   r   rB   rC   rH   rN   rP   r   rT   r   r   rZ   dictstrrc   rd   __classcell__r+   r+   rU   r,   r   $   sJ    	
$r   )%
contextlibr   typingr   r   r   r   r#   lightning_utilitiesr   r   torch.nnr	   typing_extensionsr
   r   pytorch_lightningpl&lightning_fabric.plugins.precision.ampr   'lightning_fabric.plugins.precision.fsdpr   (lightning_fabric.plugins.precision.utilsr   r    lightning_fabric.utilities.typesr   -pytorch_lightning.plugins.precision.precisionr   &pytorch_lightning.utilities.exceptionsr   r>   r   r:   r"   r   r   r+   r+   r+   r,   <module>   s$   