o
    8wiD                  	   @   s&  U d dl Z d dlmZmZmZ d dlmZ d dlmZm	Z	 d dl
mZ d dlmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/ ede0e1ef dZ2dZ3dZ4e5e6d< G dd dZ7G dd de-Z8G dd  d Z9d!ed"efd#d$Z:d%eeef d"e;eeej<f ee0e1ef  f fd&d'Z=d(ej<d)e0e1ef d"efd*d+Z>d,e5d-ed"dfd.d/Z?d%e@d"e5fd0d1ZAd2ed"efd3d4ZBeBejCe_CdS )5    N)	GeneratorIteratorMapping)deepcopy)partialwraps)
MethodType)AnyCallableOptionalTypeVarUnionoverload)is_overridden)apply_to_collection)Tensor)nn)OptimizedModule)_IncompatibleKeys)	Optimizer)
DataLoader)override)	Precision)Strategy)move_data_to_device)_set_sampler_epoch)_DeviceDtypeModuleMixin)OptimizableT_destination)bound)training_stepvalidation_step	test_steppredict_stepF_in_fabric_backwardc                	   @   s   e Zd Zddededeee  ddfddZe	defdd	Z
deeef fd
dZdeeef ddfddZddee defddZdedefddZdS )_FabricOptimizerN	optimizerstrategy	callbacksreturnc                 C   s8   || _ || _|p	g | _td|jj | j|jfi | _dS )a  FabricOptimizer is a thin wrapper around the :class:`~torch.optim.Optimizer` that delegates the optimizer
        step calls to the strategy.

        The underlying wrapped optimizer object can be accessed via the property :attr:`optimizer`.

        Args:
            optimizer: The optimizer to wrap
            strategy: Reference to the strategy for handling the optimizer step

        FabricN)
_optimizer	_strategy
_callbackstype	__class____name__)selfr&   r'   r(    r2   V/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/lightning_fabric/wrappers.py__init__5   s   
"z_FabricOptimizer.__init__c                 C      | j S N)r+   r1   r2   r2   r3   r&   F      z_FabricOptimizer.optimizerc                 C   s   | j | jS r6   )r,   get_optimizer_stater&   r7   r2   r2   r3   
state_dictJ   s   z_FabricOptimizer.state_dictr:   c                 C   s   | j | d S r6   )r&   load_state_dict)r1   r:   r2   r2   r3   r;   M   s   z _FabricOptimizer.load_state_dictclosurec                 C   s   |d urd|ini }t | jdrt| jjtr| jj}n| j}| jj|fi |}| jD ]}t|dd }t	|r?|| j|d q,|S )Nr<   modelon_after_optimizer_step)r'   r&   )
hasattrr,   
isinstancer=   r   r&   optimizer_stepr-   getattrcallable)r1   r<   kwargsr&   outputcallbackhookr2   r2   r3   stepP   s   

z_FabricOptimizer.stepitemc                 C   s   t | j|S r6   )rB   r+   r1   rI   r2   r2   r3   __getattr__a   s   z_FabricOptimizer.__getattr__r6   )r0   
__module____qualname__r   r   r   listr
   r4   propertyr&   dictstrr   r:   r;   r	   rH   rK   r2   r2   r2   r3   r%   4   s    $r%   c                       s  e Zd Z	d0dejdedeej ddf fddZedejfdd	Z	e
d
ededefddZeddddedededefddZeddddededeeef fddZe
	d1dee dededeeeef  fddZe
	d2deeef dededefddZdeeef ddfdd Zd!edefd"d#Zded$edefd%d&Zd'edefd(d)Ze
d*edef fd+d,Ze
d$ed-eddf fd.d/Z  Z S )3_FabricModuleNforward_moduler'   original_moduler)   c                    s4   t    || _|p|| _|| _tt| _d| _dS )a  The FabricModule is a thin wrapper around the :class:`torch.nn.Module` and handles precision / autocast
        automatically for the forward pass.

        The underlying wrapped module can be accessed via the property :attr:`module`.

        Args:
            forward_module: The module to wrap the ``forward`` method on.
            strategy: Reference to the strategy for handling precision etc.
            original_module: The original, unmodified module as passed into the
                :meth:`lightning_fabric.fabric.Fabric.setup` method. This is needed when attribute lookup
                on this wrapper should pass through to the original module.

        TN)	superr4   _forward_module_original_moduler,   set_LIGHTNING_MODULE_STEP_METHODS_forward_methods_fabric_module_initialized)r1   rS   r'   rT   r/   r2   r3   r4   f   s   



z_FabricModule.__init__c                 C   s   | j p| jS r6   )rW   rV   r7   r2   r2   r3   module}   s   z_FabricModule.moduleargsrD   c                 O   sp   | j j}|||f\}}|  | j|i |}W d   n1 s$w   Y  ||}t|t| jd |S )ziCasts all inputs to the right precision and handles autocast for operations in the module forward method.Ndtypefunction)	r,   	precisionconvert_inputforward_contextrV   convert_outputr   r   _register_backward_hook)r1   r^   rD   rb   rE   r2   r2   r3   forward   s   

z_FabricModule.forward.)prefix	keep_varsdestinationrh   ri   c                C      d S r6   r2   r1   rj   rh   ri   r2   r2   r3   r:         z_FabricModule.state_dictc                C   rk   r6   r2   )r1   rh   ri   r2   r2   r3   r:      rm    Fc                 C   s   | j j|||dS )N)rj   rh   ri   )rW   r:   rl   r2   r2   r3   r:      s
   Tr:   strictc                 K   s   | j jd||d|S )N)r:   ro   r2   )rW   r;   )r1   r:   ro   rD   r2   r2   r3   r;      s   z_FabricModule.load_state_dictmethodc                 C   s   t |ttfstdt|j t |tr|n|j}|dkr#tdt t| j|dts?t	d| dt| jj d| d| j
| dS )	z]Mark a method as a 'forward' method to prevent it bypassing the strategy wrapper (e.g., DDP).z(Expected a method or a string, but got: rg   z>You cannot mark the forward method itself as a forward method.NzYou marked 'z' as a forward method, but `.z$` does not exist or is not a method.)r@   r   rQ   	TypeErrorr.   r0   
ValueErrorrB   rW   AttributeErrorrZ   add)r1   rp   namer2   r2   r3   mark_forward_method   s   z!_FabricModule.mark_forward_methodmethod_namec                    sR    dksJ j jdtdtdtf fdddtdtdtffdd}|S )	Nrg   r^   rD   r)   c                     s"   j _tj  }|| i |S r6   )rW   rg   rB   )r^   rD   rp   )rx   original_forwardr1   r2   r3   wrapped_forward   s   zC_FabricModule._redirection_through_forward.<locals>.wrapped_forwardc                     s    j _ j| i |S r6   )rW   rg   )r^   rD   )r1   rz   r2   r3   call_forward_module   s   zG_FabricModule._redirection_through_forward.<locals>.call_forward_module)rW   rg   r	   )r1   rx   r{   r2   )rx   ry   r1   rz   r3   _redirection_through_forward   s
   z*_FabricModule._redirection_through_forwardrv   c                    sL   ddt dt ddffdd tdt d	t dt f fd
d}|S )zTracks whether any submodule in ``self._original_module`` was called during the execution of ``method`` by
        registering forward hooks on all submodules.F___r)   Nc                     s   d d S )NTr2   )r}   r~   )module_calledr2   r3   rG      s   zA_FabricModule._wrap_method_with_module_call_tracker.<locals>.hookr^   rD   c                     sp   g }j  D ]
}||  q| i |}r-tdtj j d dd|D ]}|  q/|S )NzYou are calling the method `rq   z()` from outside the model. To avoid issues with the currently selected strategy, explicitly mark it as a forward method with `fabric_model.mark_forward_method(z)` after `fabric.setup()`.)rW   modulesappendregister_forward_hookRuntimeErrorr.   r0   remove)r^   rD   handlesr]   rE   handlerG   rp   r   rv   r1   r2   r3   _wrapped_method   s   
zL_FabricModule._wrap_method_with_module_call_tracker.<locals>._wrapped_method)r	   r   )r1   rp   rv   r   r2   r   r3   %_wrap_method_with_module_call_tracker   s
   $z3_FabricModule._wrap_method_with_module_call_trackertensorc                    sL   |j s|S td jtd}t fdddD }tt|p|}|| |S )Nbackwardparentc                 3   s"    | ]}t | jjtd V  qdS )r   N)r   r,   rb   r   ).0rp   r7   r2   r3   	<genexpr>   s
    
z8_FabricModule._register_backward_hook.<locals>.<genexpr>)pre_backwardr   post_backward)requires_gradr   r,   r   anyr   _backward_hookregister_hook)r1   r   strategy_requiresprecision_requiresrG   r2   r7   r3   rf      s   
z%_FabricModule._register_backward_hookrI   c                    s   |dkr|| j v r| j| jkr| |S zt |W S  tyB   t d}t||}t	|r>| j| jkr>| 
||}| Y S w )NrZ   rW   )rZ   rV   rW   r|   rU   rK   rt   rB   inspectismethodr   )r1   rI   rT   attrr\   r2   r3   rK      s   


z_FabricModule.__getattr__valuec                    s|   t | ddst || d S | j}t||}|t| v }|s)|s)t||| d S |r1t||| |r<t || d S d S )Nr[   F)rB   rU   __setattr__rW   r?   dirsetattr)r1   rv   r   rT   original_has_attrfabric_has_attrr\   r2   r3   r   
  s   
z_FabricModule.__setattr__r6   )Nrn   F)T)!r0   rL   rM   r   Moduler   r   r4   rO   r]   r   r	   rg   r   r   rQ   boolr:   rP   r   r   r;   r   r   rw   r
   r|   r   r   rf   rK   r   __classcell__r2   r2   r\   r3   rR   e   sb    $(	
$rR   c                   @   st   e Zd Zddedeej ddfddZedeej fddZde	fd	d
Z
deee eeddf f fddZdS )_FabricDataLoaderN
dataloaderdevicer)   c                 C   s$   | j |j  || _|| _d| _dS )a  The FabricDataLoader is a wrapper for the :class:`~torch.utils.data.DataLoader`. It moves the data to the
        device automatically if the device is specified.

        Args:
            dataloader: The dataloader to wrap
            device: The device to which the data should be moved. By default the device is `None` and no data
                transfers will be made (identical behavior as :class:`~torch.utils.data.DataLoader`).

        r   N)__dict__update_dataloader_device_num_iter_calls)r1   r   r   r2   r2   r3   r4   &  s   

z_FabricDataLoader.__init__c                 C   r5   r6   )r   r7   r2   r2   r3   r   5  r8   z_FabricDataLoader.devicec                 C   s
   t | jS r6   )lenr   r7   r2   r2   r3   __len__9  s   
z_FabricDataLoader.__len__c                 c   sZ    t | j| j |  jd7  _| jd u rt| jE d H  d S | jD ]	}t|| jV  q!d S )N   )r   r   r   r   iterr   rJ   r2   r2   r3   __iter__<  s   

z_FabricDataLoader.__iter__r6   )r0   rL   rM   r   r   torchr   r4   rO   intr   r   r   r	   r   r   r2   r2   r2   r3   r   %  s    (r   
collectionr)   c                 C   sN   dt tttf dt tjttf fdd}tttg}|t	 t
| t||dS )Nobjr)   c                 S   sH   t t| d  }trt|jd S t | tr| jS t | tr"| jS | S Nr   )r@   _unwrap_compiledrR   rV   r%   r&   r   r   )r   	unwrappedr2   r2   r3   _unwrapK  s   

z _unwrap_objects.<locals>._unwrapr_   )r   rR   r%   r   r   r   r   r   r   r   r   tuple)r   r   typesr2   r2   r3   _unwrap_objectsJ  s   


r   r   c                 C   s8   t | trt| dd }du rtd| j|fS | dfS )zRemoves the :class:`torch._dynamo.OptimizedModule` around the object if it is wrapped.

    Use this function before instance checks against e.g. :class:`_FabricModule`.

    _compile_kwargsNzFailed to determine the arguments that were used to compile the module. Make sure to import lightning before `torch.compile` is used.)r@   r   rB   r   	_orig_mod)r   compile_kwargsr2   r2   r3   r   \  s   

r   r]   r   c                 C   s   t j| fi |S r6   )r   compile)r]   r   r2   r2   r3   _to_compiledl  s   r   requires_backwardr}   c                 G   s   | rt s
tdd S d S )NzwThe current strategy and precision selection requires you to call `fabric.backward(loss)` instead of `loss.backward()`.)r$   r   )r   r}   r2   r2   r3   r   p  s
   r   c                 C   s   t | \} }t| tttfS )aZ  Checks if an object was set up by Fabric.

    A :class:`~torch.nn.Module` may be wrapped by a :class:`_FabricModule`, a :class:`~torch.optim.Optimizer`
    may be wrapped by a :class:`_FabricOptimizer`, or a :class:`~torch.utils.data.DataLoader` may be wrapped by
    :class:`_FabricDataLoader`.

    Args:
        obj: The object to test.

    )r   r@   rR   r%   r   )r   r}   r2   r2   r3   
is_wrappedx  s   r   
compile_fnc                    s&   t  dtdtdtf fdd}|S )aZ  Wraps the ``torch.compile`` function and captures the compile arguments.

    We extract the compile arguments so that we can reapply ``torch.compile`` in ``Fabric.setup()`` with the
    same arguments as the user passed to the original call. The arguments get stored in a dictionary
    ``_compile_kwargs`` on the returned compiled module.

    r^   rD   r)   c                     sH   | r
t | d tjs | i |S | d } |fi |}t||_|S r   )r@   r   r   r   r   )r^   rD   r=   compiled_modelr   r2   r3   _capture  s   
z)_capture_compile_kwargs.<locals>._capture)r   r	   )r   r   r2   r   r3   _capture_compile_kwargs  s   
r   )Dr   collections.abcr   r   r   copyr   	functoolsr   r   r   r   typingr	   r
   r   r   r   r   r   lightning_utilitiesr   #lightning_utilities.core.apply_funcr   r   r   torch._dynamor   torch.nn.modules.moduler   torch.optimr   torch.utils.datar   typing_extensionsr   lightning_fabric.pluginsr   lightning_fabric.strategiesr   lightning_fabric.utilitiesr   lightning_fabric.utilities.datar   -lightning_fabric.utilities.device_dtype_mixinr    lightning_fabric.utilities.typesr   rP   rQ   r   rY   r$   r   __annotations__r%   rR   r   r   r   r   r   r   r   objectr   r   r   r2   r2   r2   r3   <module>   sH   
 	1 A%8 