o
    }oi\                     @   s   d dl Z d dlZd dlmZmZ d dlZd dlZd dl	m
Z
 d dlmZmZ d dlmZ d dlmZ d dlmZ e dZG d	d
 d
eZdS )    N)AnyDict)ModelConfig)BaseModelLoader_initialize_model)set_default_torch_dtype)load_model_weights)NemoModelConfigNeMoc                   @   s   e Zd ZdZededeeef fddZde	ddfdd	Z
d
edejjfddZededefddZedeeef deeef fddZdS )NemoModelLoadera*  
    Implements a custom ModelLoader for vLLM that reads the weights from a Nemo checkpoint
    and converts them to a vLLM compatible format at load time.

    Also supports an ahead-of-time conversion that stores new weights in a Safetensors file,
    see convert_and_store_nemo_weights(...)
    	nemo_filereturnc                 C   s   t d|  d t| S )NzLoading weights from ...)LOGGERinfor   )r    r   Q/home/ubuntu/.local/lib/python3.10/site-packages/nemo/export/vllm/model_loader.py_load_nemo_checkpoint_state(   s   z+NemoModelLoader._load_nemo_checkpoint_statemodel_configNc                 C   s   t )N)NotImplementedError)selfr   r   r   r   download_model-   s   zNemoModelLoader.download_modelvllm_configc             	   C   s   |j }|j}t|tsJ t|j}t|jA t	
|j
 t|}W d   n1 s.w   Y  |j}d|v r>|d }t|}|j||}|| W d   | S 1 s\w   Y  | S )zn
        Overrides the load_model function from BaseModelLoader to convert Nemo weights at load time.
        Nconfig)r   device_config
isinstancer	   r   r   nemo_checkpointr   dtypetorchdevicer   nemo_model_config_standardize_nemo2_namingmodel_converterconvert_weightsload_weightseval)r   r   r   r   
state_dictmodelr   weights_iteratorr   r   r   
load_model0   s$   


zNemoModelLoader.load_modelsafetensors_filec                 C   s   t | tsJ tj| jsJ t| j}| j	}d|v r!|d }t
|}dd | j||D }td| d tj|| dS )z_
        Converts Nemo weights and stores the converted weights in a Safetensors file.
        r   c                 S   s   i | ]\}}||qS r   r   ).0nametensorr   r   r   
<dictcomp>^   s    zBNemoModelLoader.convert_and_store_nemo_weights.<locals>.<dictcomp>zSaving weights to r   N)r   r	   ospathexistsr'   r   r   r   r    r!   r"   r#   r   r   safetensorsr   	save_file)r   r*   r&   r   tensorsr   r   r   convert_and_store_nemo_weightsL   s   
z.NemoModelLoader.convert_and_store_nemo_weightsr&   c                 C   s   dd |   D S )Nc                 S   s   i | ]\}}| d d|qS )moduler'   )replace)r+   kvr   r   r   r.   e   s    z=NemoModelLoader._standardize_nemo2_naming.<locals>.<dictcomp>)items)r&   r   r   r   r!   c   s   z)NemoModelLoader._standardize_nemo2_naming)__name__
__module____qualname____doc__staticmethodstrr   r   r   r   r   r	   r   nnModuler)   r5   r!   r   r   r   r   r      s    
(r   )loggingos.pathr/   typingr   r   safetensors.torchr2   r   vllm.configr   'vllm.model_executor.model_loader.loaderr   r   &vllm.model_executor.model_loader.utilsr   nemo.export.utilsr   nemo.export.vllm.model_configr	   	getLoggerr   r   r   r   r   r   <module>   s   
