o
    oi                     @   s   d dl Z d dlmZmZmZmZmZmZ d dlZd dl	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ G d	d
 d
eZde
deddfddZde
deddfddZde
dejddfddZddee
ef dedefddZdS )    N)AnyCallableDictOptionalSequenceUnion)Module	Parameter)	Optimizer)TorchFunctionMode)override)rank_zero_warn)_DEVICEc                       s\   e Zd ZdZddeddf fddZe		dd	ed
edee	 de
e de	f
ddZ  ZS )
_EmptyInitzInitialize `nn.Module` with empty tensors, i.e., uninitialized memory.

    Example::

        with _EmptyInit():
            model = BigModel()
        model.load_state_dict(torch.load("checkpoint.pt"))

    TenabledreturnNc                    s   t    || _d S N)super__init__r   )selfr   	__class__ S/home/ubuntu/.local/lib/python3.10/site-packages/lightning/fabric/utilities/init.pyr   '   s   

z_EmptyInit.__init__r   functypesargskwargsc                 C   sR   |pi }| j s||i |S t|dd dkr"d|v r|d S |d S ||i |S )N
__module__ztorch.nn.inittensorr   )r   getattr)r   r   r   r   r   r   r   r   __torch_function__+   s   z_EmptyInit.__torch_function__T)r   N)__name__r   __qualname____doc__boolr   r   r   r   r   r   r   r!   __classcell__r   r   r   r   r      s"    
r   moduledevicer   c                 C   s:   | j |dd t| dstdt| j d|   dS )zMaterialize a module.F)r)   recursereset_parametersz#Materialization requires that the `z~.reset_parameters` method is implemented. This method is used to initialize any children parameters or buffers in this module.N)to_emptyhasattr	TypeErrortyper#   r+   r(   r)   r   r   r   _materialize=   s   
r1   c                 C   s(   |   D ]} t| ddrt| | qdS )z*Materialize all tensors in a given module.Fr*   N)modules&_has_meta_device_parameters_or_buffersr1   r0   r   r   r   _materialize_meta_tensorsH   s
   
r5   c              	   C   s   t | sd S | j|d t }|  D ].}tdd t|jdd|jddD r+qt	t
|dd  }r9|  q|t|j q|rPtdd|  d S d S )	N)r)   c                 s   s    | ]}d V  qdS )FNr   ).0_r   r   r   	<genexpr>Z   s    z2_materialize_distributed_module.<locals>.<genexpr>Fr2   r+   zParameter initialization incomplete. The following modules have parameters or buffers with uninitialized memory because they don't define a `reset_parameters()` method for re-initialization: z, )r4   r,   setr3   all	itertoolschain
parametersbufferscallabler    addr/   r#   r   join)r(   r)   uninitialized_modules	submodulereset_methodr   r   r   _materialize_distributed_moduleO   s"   *rE   Tobjr*   c                 C   sf   t | trtdd | jD S t | tr)tdd t| j|d| j|dD S t	dt
| j )Nc                 s   s.    | ]}|d  D ]}t |tr|jV  qqdS )paramsN)
isinstancer	   is_meta)r6   param_grouptr   r   r   r8   l   s    
z9_has_meta_device_parameters_or_buffers.<locals>.<genexpr>c                 s   s    | ]}|j V  qd S r   )rI   )r6   rK   r   r   r   r8   p   s    r2   z<Expected `torch.nn.Module` or `torch.optim.Optimizer`, got: )rH   r
   anyparam_groupsr   r;   r<   r=   r>   r.   r/   r#   )rF   r*   r   r   r   r4   j   s   

*r4   r"   )r;   typingr   r   r   r   r   r   torchtorch.nnr   r	   torch.optimr
   torch.overridesr   typing_extensionsr   $lightning.fabric.utilities.rank_zeror    lightning.fabric.utilities.typesr   r   r1   r5   r)   rE   r&   r4   r   r   r   r   <module>   s    !$