o
    8wi                     @   s   d dl Z d dlmZ d dlmZmZmZmZ d dlZd dl	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ G d
d deZde
deddfddZde
deddfddZde
dejddfddZddee
ef dedefddZdS )    N)Sequence)AnyCallableOptionalUnion)Module	Parameter)	Optimizer)TorchFunctionMode)override)rank_zero_warn)_DEVICEc                       s\   e Zd ZdZddeddf fddZe		dd	ed
edee	 de
e de	f
ddZ  ZS )
_EmptyInitzInitialize `nn.Module` with empty tensors, i.e., uninitialized memory.

    Example::

        with _EmptyInit():
            model = BigModel()
        model.load_state_dict(torch.load("checkpoint.pt"))

    TenabledreturnNc                    s   t    || _d S N)super__init__r   )selfr   	__class__ \/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/lightning_fabric/utilities/init.pyr   (   s   

z_EmptyInit.__init__r   functypesargskwargsc                 C   sR   |pi }| j s||i |S t|dd dkr"d|v r|d S |d S ||i |S )N
__module__ztorch.nn.inittensorr   )r   getattr)r   r   r   r   r   r   r   r   __torch_function__,   s   z_EmptyInit.__torch_function__T)r   N)__name__r   __qualname____doc__boolr   r   r   r   r   r   dictr    __classcell__r   r   r   r   r      s"    
r   moduledevicer   c                 C   sH   | j |dd t| dstdt| j dt| jr"|   dS dS )zMaterialize a module.F)r)   recursereset_parametersz#Materialization requires that the `z~.reset_parameters` method is implemented. This method is used to initialize any children parameters or buffers in this module.N)to_emptyhasattr	TypeErrortyper"   callabler+   r(   r)   r   r   r   _materialize>   s   

r2   c                 C   s(   |   D ]} t| ddrt| | qdS )z*Materialize all tensors in a given module.Fr*   N)modules&_has_meta_device_parameters_or_buffersr2   r1   r   r   r   _materialize_meta_tensorsJ   s
   
r6   c              	   C   s   t | sd S | j|d t }|  D ].}tdd t|jdd|jddD r+qt	t
|dd  }r9|  q|t|j q|rPtdd|  d S d S )	N)r)   c                 s   s    | ]}d V  qdS )FNr   ).0_r   r   r   	<genexpr>\   s    z2_materialize_distributed_module.<locals>.<genexpr>Fr3   r+   zParameter initialization incomplete. The following modules have parameters or buffers with uninitialized memory because they don't define a `reset_parameters()` method for re-initialization: z, )r5   r,   setr4   all	itertoolschain
parametersbuffersr0   r   addr/   r"   r   join)r(   r)   uninitialized_modules	submodulereset_methodr   r   r   _materialize_distributed_moduleQ   s"   *rE   Tobjr*   c                 C   sf   t | trtdd | jD S t | tr)tdd t| j|d| j|dD S t	dt
| j )Nc                 s   s.    | ]}|d  D ]}t |tr|jV  qqdS )paramsN)
isinstancer   is_meta)r7   param_grouptr   r   r   r9   n   s    
z9_has_meta_device_parameters_or_buffers.<locals>.<genexpr>c                 s   s    | ]}|j V  qd S r   )rI   )r7   rK   r   r   r   r9   r   s    r3   z<Expected `torch.nn.Module` or `torch.optim.Optimizer`, got: )rH   r	   anyparam_groupsr   r<   r=   r>   r?   r.   r/   r"   )rF   r*   r   r   r   r5   l   s   

*r5   r!   )r<   collections.abcr   typingr   r   r   r   torchtorch.nnr   r   torch.optimr	   torch.overridesr
   typing_extensionsr   $lightning_fabric.utilities.rank_zeror    lightning_fabric.utilities.typesr   r   r2   r6   r)   rE   r%   r5   r   r   r   r   <module>   s   !$