o
    پi                     @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlmZ ddlmZmZmZmZmZmZmZmZmZ ddlZddlZddlZddlZddlZddlmZm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( ddl)m*Z* dd	l+m,Z, dd
l-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9 ddl:m;Z;m<Z< ddl=m>Z>m?Z?m@Z@mAZAmBZB ddlCmDZD z
ddlEmFZFmGZG W n eHy ZI z
d ZFZGW Y dZI[IndZI[Iww e	JeKZLdd ZMeM  eN ZO	ddePdeeP dePfddZQdd ZRdePdePddfd d!ZSd"ePd#eTePePf dePfd$d%ZUd"ePd&eTePePf dePfd'd(ZVG d)d* d*e(ZW	dd+e,d,e*d-eePeeP f d.eePePf dB de3f
d/d0ZXd1ePdeeYeeP f fd2d3ZZ	ddePdeeP d4eeP d5eeP deeP f
d6d7Z[			8ddePdeeP d4eeP d5eeP d9eeePeeP f  d:e\dePfd;d<Z]	ddePd=ePdeeP d5eeP ddf
d>d?Z^d@eeP dAePd=ePdeeP fdBdCZ_d@eeP dAePd=ePdeeP fdDdEZ`d@eeP deeP fdFdGZadePdeeP dAePd@eeP deeePejbf ddf f
dHdIZc	Jdd@eeP dKeYdeeePejbf ddf fdLdMZdd@eeP deeePejbf ddf fdNdOZe	Jdd@eeP dPe\dKeYdeeePejbf ddf fdQdRZf	Jdd@eeP dPe\dKeYdeeePejbf ddf fdSdTZgdUePdeTfdVdWZhd@eeP deeePejbf ddf fdXdYZid@eeP dPe\deeePejbf ddf fdZd[Zjd\ePd]eePePf deeP fd^d_Zkd\ePd]eePePf deeePejbf ddf fd`daZldbedejbfdcddZmdeejbdfejbddfdgdhZndeejbdfejbddfdidjZoeejbejbgejbf Zpdke\depfdldmZqdnepdoeejbgejbf depfdpdqZrd@eeP deeePejbf ddf fdrdsZsd,e*fdtduZt	v	w	xddyejujvdzewd{ewd|e\ddf
d}d~ZxdePdeTdeeP fddZyG dd de#ZzG dd de#Z{dePde\de\de\deeP deee\ewf  fddZ|dd Z}dd Z~	dddZdS )z9Utilities for downloading and initializing model weights.    N)defaultdict)	AnyCallableDict	GeneratorIterableListOptionalTupleUnion)HfFileSystemhf_hub_downloadsnapshot_download)	BaseModel
ConfigDictValidationInfomodel_validator)tqdm)
LoadConfig)ModelConfig)get_tensor_model_parallel_rank$get_tensor_model_parallel_world_size)get_attention_tp_rank)QuantizationConfigget_quantization_config)	Fp8Config)ModelOptFp4ConfigModelOptFp8Config)%ci_download_with_validation_and_retry&ci_validate_and_cleanup_local_snapshot)
BAR_FORMATfind_local_repo_diris_cpulog_info_on_rank0print_warning_once)is_in_ci)SafeTensorsFileLoaderSingleGroupc                  C   s:   dt jvrzddl} dtj_W dS  ty   Y dS w dS )z#automatically activates hf_transferHF_HUB_ENABLE_HF_TRANSFERr   NT)osenvironhf_transferhuggingface_hub	constantsr(   ImportError)r+    r/   X/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/model_loader/weight_utils.pyenable_hf_transferG   s   
r1    model_name_or_path	cache_dirsuffixc                 C   sh   |pt }tjtj|dd | dd}t| 	 }|| | d }t
jtj||dd}|S )NTexist_ok/-z.locki  mode)temp_dirr)   makedirspathdirnamereplacehashlibsha256encode	hexdigestfilelockFileLockjoin)r3   r4   r5   lock_dir
model_name	hash_namelock_file_namelockr/   r/   r0   get_lock]   s   rM   c                 C   s\   t t}|  D ]\}}||  | qg }| D ]\}}t|dkr+|| q|S N   )r   listitemsdata_ptrappendlen)tensorsptrskvfailing_namesr/   r/   r0   _shared_pointersk   s   
r\   pt_filenamesf_filenamereturnc              
   C   s&  t j| ddd}d|v r|d }t|}|D ]}|dd  D ]}|| qqdd | D }tj|}tj|dd d	d
l	m
} |||ddid t|j}t| j}	||	 |	 dkrqtd| d| d|  d|	 d	tj |}
|D ]}|| }|
| }t ||std| qyd S )NcpuTmap_locationweights_only
state_dictrO   c                 S   s   i | ]	\}}||  qS r/   )
contiguous).0rW   rX   r/   r/   r0   
<dictcomp>   s    z2convert_bin_to_safetensor_file.<locals>.<dictcomp>r6   r   )	save_fileformatpt)metadatag{Gz?z4The file size different is more than 1%:
         - : z
         - z

         z(The output tensors do not match for key )torchloadr\   poprQ   r)   r>   r?   r=   safetensors.torchrh   statst_sizeRuntimeErrorsafetensors	load_fileequal)r]   r^   loadedsharedshared_weightsnamer?   rh   sf_sizept_sizereloadedrW   	pt_tensor	sf_tensorr/   r/   r0   convert_bin_to_safetensor_filev   sB   
r   keyprefix_mappingc                 C   s.   |  D ]\}}| |r| ||d} q| S rN   )rQ   
startswithr@   )r   r   prefix
new_prefixr/   r/   r0   replace_prefix   s
   
r   substring_mappingc                 C   s*   |  D ]\}}|| v r| ||} q| S N)rQ   r@   )r   r   substr
new_substrr/   r/   r0   replace_substrings   s
   r   c                       s   e Zd Z fddZ  ZS )DisabledTqdmc                    s   d|d< t  j|i | d S )NTdisable)super__init__)selfargskwargs	__class__r/   r0   r      s   zDisabledTqdm.__init__)__name__
__module____qualname__r   __classcell__r/   r/   r   r0   r      s    r   model_configload_configpacked_modules_mappingremap_prefixc              	      s(  t | j}| jdkr|i S t| jdd }t| jdd }|d u r+|d ur+t|dd }|d u r6t| jdd }|d urLt|tsC| }||d< ||S | jdkrf|jrYd|jvr`|dd	iS |jd }n| j	}t
j|}|st||j t|| jd
|jtjjtd}	W d    n1 sw   Y  n|}	|   s| jdkrtdddS | S tt
j|	d
}
 fdd|
D }t|dkrtd| j t|dkrtd| j d| |d }t|}t|}d urfdd|d d D }||d d< ||d< | jdkr||d< nj| jdr}|di dd	dr}|d d }|d u rU| jj d dkrLtd| j d | jj d  d!	 W d    d S |d"ks`| jd#krlt!|W  d    S d$|v r}t"|W  d    S ||W  d    S 1 sw   Y  d S )%Nggufquantization_configtext_configcompression_configr   bitsandbytesqlora_adapter_name_or_pathadapter_name_or_pathr2   z*.json)revisionallow_patternsr4   local_files_only
tqdm_classmxfp8TF)	use_mxfp8is_checkpoint_fp8_serializedc                    s&   g | ] t  fd dD r qS )c                 3       | ]}  |V  qd S r   endswithrf   xfr/   r0   	<genexpr>       z.get_quant_config.<locals>.<listcomp>.<genexpr>anyrf   )possible_config_filenamesr   r0   
<listcomp>   
    z$get_quant_config.<locals>.<listcomp>r   z Cannot find the config file for rO   z Found multiple config files for rl   c                    s   g | ]}t | qS r/   )r   rf   r   )r   r/   r0   r     s    quantizationexclude_modulesmodeloptproducerrz   
quant_algoLlamaForCausalLMEagle3z+Invalid quant_config, quantization method: z,hf architectures: z. FP8modelopt_fp8FP4)#r   r   from_configgetattr	hf_config
isinstancedictto_dictmodel_loader_extra_config
model_pathr)   r>   isdirrM   download_dirr   r   r,   r-   HF_HUB_OFFLINEr   get_config_filenamesr   globrG   rT   
ValueErroropenjsonrn   r   getarchitecturesr   r   )r   r   r   r   	quant_clshf_quant_confighf_text_configr3   is_local	hf_folderconfig_filesquant_config_filesquant_config_filer   configr   r   r/   )r   r   r0   get_quant_config   s   




















&r   snapshot_dirc           	         s   dd t  D }|sdS |D ]}}t j |}t j|s qzTt|}t|di }W d   n1 s9w   Y  |sBW qt	|
 } fdd|D }|rsddt| d	| d
|dd  t|dkrkdnd fW   S W q ty } ztd|| W Y d}~qd}~ww dS )ad  
    Check if all files listed in safetensors index files actually exist on disk.

    This catches cases where the snapshot directory exists but files are missing
    (e.g., due to incomplete downloads or corrupted cache).

    Args:
        snapshot_dir: Path to the model snapshot directory

    Returns:
        Tuple of (all_exist, error_message)
    c                 S   s   g | ]	}| d r|qS )z.safetensors.index.jsonr   rf   r   r/   r/   r0   r   ,  s
    
z,_check_index_files_exist.<locals>.<listcomp>)TN
weight_mapNc                    s&   g | ]}t jt j |s|qS r/   )r)   r>   existsrG   )rf   fnr   r/   r0   r   =  s    FzMissing z file(s) from index rl      z...r2   z Failed to read index file %s: %s)r)   listdirr>   rG   r   r   r   rn   r   setvaluesrT   	Exceptionloggerwarning)	r   index_files
index_file
index_pathr   r   required_filesmissing_fileser/   r   r0   _check_index_files_exist  sD   



r   r   r   c              
   C   s  t j| rdS d}|rxzRt j|tjjdg| d}|}|sKt j|dd}t j|rKt	|}|
  }W d   n1 sFw   Y  |r]t j|d|}	t j|	r]|	}W n tyw }
 ztd||
 W Y d}
~
nd}
~
ww |szt| |}	|	rt j|	r|	}W n ty }
 ztd|
 W Y d}
~
nd}
~
ww |du rdS t j|sdS g }z"|D ]}tt j||}|D ]}t j|sq|| qqW n ty }
 ztd	|||
 g }W Y d}
~
nd}
~
ww |rt|\}}|sttd
|  d| d dS t r|rt| ||}|sdS t|dkr4ttd|  d| d |S ttd| d| d dS )a  Find local HF snapshot directory without locking.

    IMPORTANT: Caller MUST hold the model lock before calling this function
    to prevent race conditions during validation and cleanup.

    If the weights are already local, skip downloading and returns the path.
    Nmodelsr8   refsmain	snapshotsz8Failed to find local snapshot in custom cache_dir %s: %sz5Failed to find local snapshot in default HF cache: %sz5Failed to scan local snapshot %s with patterns %s: %szLocal snapshot incomplete for rl   z. Will download missing files.r   zFound local HF snapshot for z at z; skipping download.zLocal HF snapshot at z has no files matching z; will attempt download.)r)   r>   r   rG   r,   r-   REPO_ID_SEPARATORsplitisfiler   readstripr   r   r   r!   r   r   rS   r   r#   r%   r   rT   )r3   r4   r   r   found_local_snapshot_dirrepo_folder
rev_to_useref_mainr   rev_dirr   local_weight_filespatternmatched_filesis_complete	error_msgis_validr/   r/   r0   $_find_local_hf_snapshot_dir_unlockedO  s   

r  r   ignore_patternsmax_retriesc              
   C   s  t j| r| S t| |l t| |||}|dur"|W  d   S tjjsGt }|j	| d|d}|D ]}	t
||	}
t|
dkrF|	g} nq3ttd|  t sht| |||t|tjjd}|W  d   S t| |||||dW  d   S 1 s|w   Y  dS )aA  Download model weights from Hugging Face Hub.

    Args:
        model_name_or_path (str): The model name or path.
        cache_dir (Optional[str]): The cache directory to store the model
            weights. If None, will use HF defaults.
        allow_patterns (List[str]): The allowed patterns for the
            weight files. Files matched by any of the patterns will be
            downloaded.
        revision (Optional[str]): The revision of the model.
        ignore_patterns (Optional[Union[str, List[str]]]): The patterns to
            filter out the weight files. Files matched by any of the patterns
            will be ignored.
        max_retries (int): Maximum number of download retries if corruption
            is detected. Defaults to 3.

    Returns:
        str: The path to the downloaded model weights.
    NF)detailr   r   zUsing model weights format )r   r	  r4   r   r   r   )r3   r   r	  r4   r   r
  )r)   r>   r   rM   r  r,   r-   r   r   lsfnmatchfilterrT   r#   r   r%   r   r   r   )r3   r4   r   r   r	  r
  r>   fs	file_listr  matchingr   r/   r/   r0   download_weights_from_hf  sP   	'$r  r   c              	   C   s   t | |H zt| |||tjjd W n! tjjy$   td| Y n tjj	y4   td| Y nw W d   dS W d   dS W d   dS 1 sPw   Y  dS )a>  Download hf safetensors index file from Hugging Face Hub.

    Args:
        model_name_or_path (str): The model name or path.
        cache_dir (Optional[str]): The cache directory to store the model
            weights. If None, will use HF defaults.
        revision (Optional[str]): The revision of the model.
    )repo_idfilenamer4   r   r   zNo %s found in remote.zNo %s found in local cache.N)
rM   r   r,   r-   r   utilsEntryNotFoundErrorr   infoLocalEntryNotFoundError)r3   r   r4   r   r/   r/   r0   'download_safetensors_index_file_from_hf  s(   
	"r  hf_weights_filesr   c                    s   t j||}t j|s,t| dkr*|   | d dr*| d dr*| d gS | S t|}t	|d }W d    n1 sBw   Y  t
  |D ]} t j|||  qL fdd| D } | S )	N   r   zconsolidated.safetensorsrO   zmodel.safetensorsr   c                    s   g | ]}| v r|qS r/   r/   r   weight_files_in_indexr/   r0   r   [      z6filter_duplicate_safetensors_files.<locals>.<listcomp>)r)   r>   rG   r   rT   sortr   r   r   rn   r   add)r  r   r   index_file_namer   r   weight_namer/   r  r0   "filter_duplicate_safetensors_filesB  s&   

r#  c                 C   st   t |ddgd }t |dd}|dv r|dks| S tj|d}tj|r*|| v r,| S td| d | |g S )	a  
    Auto-detect and add mtp.safetensors for GLM4Moe MTP/NextN models if:
    1. mtp.safetensors exists in the model directory
    2. mtp.safetensors is NOT in the index (checkpoint packaging bug)
    3. Model architecture is Glm4MoeForCausalLM with num_nextn_predict_layers > 0

    This works around incorrectly packaged FP4 checkpoints like
    baseten-admin/glm-4.7-fp4 where mtp.safetensors exists but
    isn't referenced in model.safetensors.index.json.
    r   Nr   num_nextn_predict_layers)Glm4MoeForCausalLMGlm4MoeForCausalLMNextNzmtp.safetensorsz1Found mtp.safetensors but it's not referenced in zp. This is a checkpoint packaging bug. Auto-adding it for loading. Please report this to the checkpoint provider.)r   r)   r>   rG   r   r   r   )r  r   r   r   archnum_nextn_layersmtp_pathr/   r/   r0   maybe_add_mtp_safetensors_  s   

r*  c                    s   g d  fdd| D } | S )z
    Exclude files that are not needed for inference.

    See https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
    )ztraining_args.binzoptimizer.binzoptimizer.ptzscheduler.ptz	scaler.ptc                    s&   g | ] t  fd dD s qS )c                 3   r   r   r   r   r   r/   r0   r     r   zCfilter_files_not_needed_for_inference.<locals>.<listcomp>.<genexpr>r   r   	blacklistr   r0   r     r   z9filter_files_not_needed_for_inference.<locals>.<listcomp>r/   )r  r/   r+  r0   %filter_files_not_needed_for_inference  s
   
r-  c                 c   s   t j  pt j dk}tj|d}tj|dd tj|d}t| |x tj	|sg }t
|d| tt
 dD ]@}t j|ddd	}	|	 D ]1\}
}tj||
}t|d
}t||    W d   n1 srw   Y  ||
 qKq=t|d}t|| W d   n1 sw   Y  W d   n1 sw   Y  t|}t|}W d   n1 sw   Y  |D ]+}
tj||
}t|d}t|}W d   n1 sw   Y  |
t |fV  qdS )zIterate over the weights in the model np files.

    Will dump the model weights to numpy files if they are not already dumped.
    r   npTr6   zweight_names.jsonz"Loading np_cache checkpoint shardsdescr   
bar_formatpositionr`   ra   wbNwrb)rm   distributedis_initializedget_rankr)   r>   rG   r=   rM   r   r   r    _get_free_posrn   rQ   r   r.  saver`   detachnumpyrS   r   dump
from_numpy)r3   r4   r   r  enable_tqdm	np_folderweight_names_fileweight_namesbin_filestaterz   param
param_pathr   r/   r/   r0   np_cache_weights_iterator  sN   

rG  Fdisable_mmapc              	   c   s    t j  pt j dk}t| d| tt dD ]Y}|rLt|d }tj 	|
 }t| D ]	}||| fV  q2W d   n1 sFw   Y  qtj|ddd}| D ]
}|||fV  qYW d   n1 snw   Y  qdS )	7Iterate over the weights in the model safetensor files.r   z%Loading safetensors checkpoint shardsr/  r5  Nrj   r`   	frameworkdevice)rm   r6  r7  r8  r   r    r9  r   rt   rn   r   sortedkeys	safe_open
get_tensor)r  rH  r?  st_filer   resultrz   r/   r/   r0   safetensors_weights_iterator  s2   
rS  c              	   #   s   t du r	tdtj rtjjjnt z }W n t	y(   d}Y nw t
d| } fddtdt  D }d}t|dd	|d
D ]>}t |}dd t|D }|| z$| }zt|j }	|	D ]}
||
}|
|fV  qqW nw W |  qM|  w dS )z
    Iterate over the weights in the model safetensor files
    using fastsafetensor library to accelerate loading via GPU Direct Storage (if available).
    Nz@Please install fastsafetensors via `pip install fastsafetensors`r   zcuda:c                    s    g | ]} ||    qS r/   )size)rf   ir  pgr/   r0   r     s    z4fastsafetensors_weights_iterator.<locals>.<listcomp>zE{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]z/Loading safetensors using Fastsafetensor loaderF)r0  r   r1  c                 S   s   i | ]\}}||gqS r/   r/   )rf   rU  r   r/   r/   r0   rg     r  z4fastsafetensors_weights_iterator.<locals>.<dictcomp>)r&   r.   rm   r6  r7  groupWORLDr'   rankr   rL  rangerT   rT  r   	enumerateadd_filenamescopy_files_to_devicerP   key_to_rank_lidxrN  rP  close)r  rZ  rL  weight_files_sub_lists_BAR_FORMATf_listloaderrank_file_mapfbrN  rW   tr/   rV  r0    fastsafetensors_weights_iterator  sN   




rh  max_workersc           
      #   s    t j  pt j dk}dtffdd tjj|dC fdd| D }|r<ttj	|t
| d| td	}ntj	|}|D ]}| }| D ]	\}}	||	fV  qNqDW d
   d
S 1 sdw   Y  d
S )zDMulti-Thread iterate over the weights in the model safetensor files.r   rQ  c                    sN    rt | d}tj| W  d    S 1 sw   Y  tjj| ddS )Nr5  r`   rL  )r   rt   rm   rn   r   ru   )rQ  r   rH  r/   r0   
_load_file$  s
    z=multi_thread_safetensors_weights_iterator.<locals>._load_fileri  c                    s   g | ]}  |qS r/   )submit)rf   rQ  )rl  executorr/   r0   r   +  r  z=multi_thread_safetensors_weights_iterator.<locals>.<listcomp>Multi-thread loading shardstotalr0  r   r1  N)rm   r6  r7  r8  str
concurrentfuturesThreadPoolExecutorr   as_completedrT   r    rR  rQ   )
r  ri  rH  r?  ru  futures_iterfuturerd   rz   rE  r/   )rl  rH  ro  r0   )multi_thread_safetensors_weights_iterator  s,   
"rz  c              	   #   sT   t j  pt j dk}dtf fdd}|d }tjj|d~}t| }t	
 }t||D ]}	||||	 q1tt| d| tt d=}
|r| }| }~t|d	}|d	uri|||| t| D ]	}||| fV  qo~|
d |sNW d	   n1 sw   Y  W d	   d	S W d	   d	S 1 sw   Y  d	S )
u  Multi-threaded safetensor loader with bounded memory via a sliding window.

    At most (max_workers + 1) shard files are in-flight at any time:
    max_workers loading concurrently + 1 prefetched and ready to yield.
    Peak CPU RAM ≈ (max_workers + 2) × shard_file_size.
    r   rQ  c                    s   r"t | d tj  }W d    |S 1 sw   Y  |S tj| ddd  fdd  D }W d    |S 1 sAw   Y  |S )Nr5  rj   r`   rJ  c                    s   i | ]}|  |qS r/   )rP  )rf   rW   r   r/   r0   rg   S  r  zZbuffered_multi_thread_safetensors_weights_iterator.<locals>._load_file.<locals>.<dictcomp>)r   rt   rm   rn   r   rO  rN  )rQ  rR  rk  r   r0   rl  M  s   

zFbuffered_multi_thread_safetensors_weights_iterator.<locals>._load_filerO   rm  rp  )rr  r0  r   r1  r2  N)rm   r6  r7  r8  rs  rt  ru  rv  itercollectionsdeque	itertoolsislicerS   rn  r   rT   r    r9  popleftrR  nextrM  rN  update)r  ri  rH  r?  rl  buffer_sizero  	file_iterpendingrQ  pbarry  rd   	next_filerz   r/   rk  r0   2buffered_multi_thread_safetensors_weights_iterator>  sF   


"r  rC  c              
   C   sj   z	t j| dddW S  ty4 } zdt|v r/tdtj|  t j| dddW  Y d}~S  d}~ww )a`  Load a PyTorch checkpoint file, handling legacy tar format.

    PyTorch 2.6 changed the default of weights_only from False to True.
    Legacy tar format files cannot be loaded with weights_only=True.
    This function tries weights_only=True first, then falls back to False
    for legacy tar format files from trusted sources (HuggingFace Hub).
    r`   Tra   zlegacy .tar formatz6Loading %s with weights_only=False (legacy tar format)FN)	rm   rn   rs   rs  r   r   r)   r>   basename)rC  r   r/   r/   r0   _load_pt_filex  s   
r  c                 c   sV    t j  pt j dk}t| d| tt dD ]}t|}| E dH  ~qdS )z3Iterate over the weights in the model bin/pt files.r   zLoading pt checkpoint shardsr/  N)	rm   r6  r7  r8  r   r    r9  r  rQ   )r  r?  rC  rD  r/   r/   r0   pt_weights_iterator  s   
r  c                 #   s    t j  pt j dk}tjj|d;  fdd| D }|r2ttj|t	| d| t
d}ntj|}|D ]}| }| E dH  q:W d   dS 1 sSw   Y  dS )z@Multi-Thread iterate over the weights in the model bin/pt files.r   rm  c                    s   g | ]}  t|qS r/   )rn  r  )rf   rC  ro  r/   r0   r     s    z4multi_thread_pt_weights_iterator.<locals>.<listcomp>z)Multi-thread loading pt checkpoint shardsrq  N)rm   r6  r7  r8  rt  ru  rv  r   rw  rT   r    rR  rQ   )r  ri  r?  ru  rx  ry  rD  r/   r  r0    multi_thread_pt_weights_iterator  s*   

"r  	gguf_filegguf_to_hf_name_mapc                    sL   dd l }|| }t  }tdd |jD }|| } fdd|D S )Nr   c                 S   s   g | ]}|j qS r/   )rz   )rf   tensorr/   r/   r0   r     s    z/get_gguf_extra_tensor_names.<locals>.<listcomp>c                    s   g | ]} | qS r/   r/   r   r  r/   r0   r     s    )r   
GGUFReaderr   rN  rU   )r  r  r   readerexpected_gguf_keysexact_gguf_keys
extra_keysr/   r  r0   get_gguf_extra_tensor_names  s   
r  c           
      c   s    ddl }|| }|jD ]$}|j|v r1|j}||j }|jdkr1|dd}t|}||fV  q|jD ]'}|j|v r\|j}|j}||j }|jdkrR|dd}t|}	||	fV  q5dS )zf
    Iterate over the quant weights in the model gguf files and convert
    them to torch tensors
    r   NF32weightqweight_typeqweight)	r   r  rU   rz   tensor_typer@   rm   r  data)
r  r  r   r  r  weight_typerz   weight_type_namer  rE  r/   r/   r0   gguf_quant_weights_iterator  s.   












r  r   c                 C   s   t | tjs| dd } | S )a  convert PySafeSlice object from safetensors to torch.Tensor

    PySafeSlice object supports indexing, which is done before loading the
    actual tensor and can reduce the amount of memory being read into the
    memory. However, it does not support more advanced functionalities
    like `.view()` or `.t()`. Therefore, if we need to modify the loaded
    tensor with these more complicated operators, we need to convert to
    tensor first.
    N)r   rm   Tensor)r   r/   r/   r0   convert_pyslice_to_tensor  s   
r  rE  loaded_weightc                 C   s   z7|   dkr|  dkr| j|  W dS |  | ks/J d|  d|   d| j| W dS  ty?    w )zDefault weight loader.rO   zAttempted to load weight (z) into parameter ()N)numelr  fill_itemrT  copy_r   rE  r  r/   r/   r0   default_weight_loader  s   r  c                 C   sN   t  }|  dkrdnd}|dur"| jj| }|| }||||}t| |S )z'Load weights that are row-parallelized.rO   r   N)r   dimr  shapenarrowr  )rE  r  tp_rank	shard_dim
shard_size	start_idxr/   r/   r0   row_parallel_weight_loader  s   
r  
shard_axisc                    s"   dt jdt jddf fdd}|S )zCCreate a weight loader that shards the weights along the given axisrE  r  r_   Nc                    s~   t  }| jj  }|| }t r3|dt  dkr3| dkr3| j}t||d| |\}}t||S |	 ||}t| |S )Nr   rO   )
r   r  r  r"   rT  r   r  %narrow_padded_param_and_loaded_weightr  r  )rE  r  r  r  r  
param_datar  r/   r0   rd  $  s&   

z%sharded_weight_loader.<locals>.loaderrm   r  )r  rd  r/   r  r0   sharded_weight_loader!  s   r  rd  r   c                    s$   dt jdt jddf fdd}|S )zDCreate a weight loader that post-processes the weights after loadingrE  r  r_   Nc                    s   | | | j  |  d S r   )r  r  r  r   rd  r/   r0   composed_loaderE  s   
z/composed_weight_loader.<locals>.composed_loaderr  )rd  r   r  r/   r  r0   composed_weight_loader@  s    r  c                 c   s    ddl m} tj  ptj dk}| $}t| d| tt dD ]}|	| |
 E dH  q$W d   dS 1 s>w   Y  dS )rI  r   )SafetensorsStreamerz.Loading safetensors using Runai Model Streamerr/  N)runai_model_streamerr  rm   r6  r7  r8  r   r    r9  stream_fileget_tensors)r  r  r?  streamerrQ  r/   r/   r0   "runai_safetensors_weights_iteratorM  s    

"r  c                 C   s   | j r2| j }d|v rt|dtrt|dtjd< d|v r2t|dtr2t|dtjd< td}td}|d u rK|d urM|tjd< d S d S d S )NconcurrencyRUNAI_STREAMER_CONCURRENCYmemory_limitRUNAI_STREAMER_MEMORY_LIMITRUNAI_STREAMER_S3_ENDPOINTAWS_ENDPOINT_URL)r   r   r   intrs  r)   r*   getenv)r   extra_configrunai_streamer_s3_endpointaws_endpoint_urlr/   r/   r0   set_runai_streamer_envc  s&   







r  MbPMbP?  modellowhighseedc                 C   s   |    D ]C}t|rItj|jjd}|| t|jj	j
dk rA|jj	}|jtj}|j|||d|}|j| q|j|||d qdS )as  Initialize model weights with random values.

    The model weights must be randomly initialized for accurate performance
    measurements. Additionally, the model weights should not cause NaNs in the
    forward pass. We empirically found that initializing the weights with
    values between -1e-3 and 1e-3 works well for most models.

    We use per-parameter random seed, so that dummy weights are consistent,
    even if the model is partitioned across multiple devices. When the seed
    is fixed, the random values generated by this function only depends on
    the parameter's number of elements and its data type.
    rj     )	generatorN)rd   r   rm   is_floating_pointr   r  rL  manual_seedfinfodtypebitstofloat16uniform_r  )r  r  r  r  rE  r  r  	tmp_paramr/   r/   r0   initialize_dummy_weights{  s   

r  rz   params_dictc                 C   sR  |  dr"td | dd}||vr td|  d| d dS |S dd	g}d
dg}|D ]]}|  |rd}|D ]}| |d  d| | v rJ|} nq7|durd| | |d  d| | d| }n	| |d| }||vrtd| d|  d| d| d	  dS |  S q,ddddd}| D ]\}	}
|  |	r| |	|
  S q| S )a  Remap the name of FP8 k/v_scale parameters.

    This function handles the remapping of FP8 k/v_scale parameter names.
    It detects if the given name ends with a suffix and attempts to remap
    it to the expected name format in the model. If the remapped name is not
    found in the params_dict, a warning is printed and None is returned.

    Args:
        name (str): The original loaded checkpoint parameter name.
        params_dict (dict): Dictionary containing the model's named parameters.

    Returns:
        str: The remapped parameter name if successful, or the original name
             if no remapping is needed.
        None: If the remapped name is not found in params_dict.
    z	.kv_scalezDEPRECATED. Found kv_scale in the checkpoint. This format is deprecated in favor of separate k_scale and v_scale tensors and will be removed in a future release. Functionally, we will remap kv_scale to k_scale and duplicate k_scale to v_scalez.attn.k_scalez'Found kv_scale in the checkpoint (e.g. z6), but not found the expected name in the model (e.g. z). kv_scale is not loaded.Nz.k_scalez.v_scalez.self_attn.z.mixer.rO   _projattnz.attnzFound z in the checkpoint (e.g. z). z is not loaded.z.attn.q_scalez.attn.v_scalez.attn.prob_scale)z.q_proj.output_scalez.k_proj.output_scalez.v_proj.output_scalezself_attn.prob_output_scale)r   r$   r@   rQ   )rz   r  remapped_namepossible_scale_namesmodelopt_attn_prefixes
scale_namematched_prefixattn_prefixquark_scale_namesquark_scale_namesglang_scale_namer/   r/   r0   maybe_remap_kv_scale_name  sd   


r  c                   @   sz   e Zd ZU eed< eeeeef f ed< edddddZ	eddde
dd fd	d
Zeddde
dd fddZdS )KVCacheQuantSchemar  scaling_factorafterr:   r_   c                 C   s    | j dksJ d| j  d| S )Nfloat8_e4m3fnz5Loaded scaling factors intended for KV cache dtype = z rather than float8_e4m3fn!)r  )r   r/   r/   r0   check_is_fp8  s   zKVCacheQuantSchema.check_is_fp8r  c              	   C   s   |j }|rV|d }|d }t| j|ks"J dt| j d| d| j D ]\}}t||ksAJ d| d| dt| dq't|D ]}|| jv sUJ d| d	qF| S )
Ntp_sizenum_hidden_layerszLoaded dictionary has TP size z2 but LLM engine is currently running with TP size .z KV cache scales map for TP rank z is malformed. Expected z layers, got z not found.)contextrT   r  rQ   r[  )r   r  r  r  r  r  
layer_mapsrU  r/   r/   r0   check_tp_ranks  s,   
z!KVCacheQuantSchema.check_tp_ranksc                 C   sT   |j }|r(|d }|d }| j| }t|D ]}||v s'J d| d| dq| S )Nr  r  z)Could not find KV cache scales for layer z in TP rank r  )r  r  r[  )r   r  r  r  r  layer_scales_maprU  r/   r/   r0   check_current_rank  s   

z%KVCacheQuantSchema.check_current_rankN)r_   r  )r   r   r   rs  __annotations__r   r  floatr   r  r   r  r  r/   r/   r/   r0   r    s   
 r  c                   @   sH   e Zd ZU eddZee ed< eed< e	ddde
dd fd	d
ZdS )QuantParamSchemar/   )protected_namespaces
model_typekv_cacher  r:   r  r_   c                 C   sD   |j }|r |dd }|d ur || jks J d| d| j d| S )Nr  zModel type is z> but loaded scaling factors belonging to different model type !)r  r   r  )r   r  r  r  r/   r/   r0   check_model_type)  s   z!QuantParamSchema.check_model_typeN)r   r   r   r   r   r	   rs  r  r  r   r   r  r/   r/   r/   r0   r  "  s   
 
r  r  r  r  r  r  c           
      C   s   z4t | &}||||d}t|}tj||d}|jj| }	|	 W  d   W S 1 s.w   Y  W n, tyC   t	
d|  Y n tjyR   t	
d|  Y n ty`   t	
d|  Y nw t	d| g S )ad  
    A simple utility to read in KV cache scaling factors that have been
    previously serialized to disk. Used by the model to populate the appropriate
    KV cache scaling factors. The serialization should represent a dictionary
    whose keys are the TP ranks and values are another dictionary mapping layers
    to their KV cache scaling factors.
    )r  r  r  r  )r  Nz!File or directory '%s' not found.z!Error decoding JSON in file '%s'.z%An error occurred while reading '%s'.znDefaulting to KV cache scaling factors = 1.0 for all layers in TP rank %d as an error occurred during loading.)r   r   rn   r  model_validater   r  rQ   FileNotFoundErrorr   errorJSONDecodeErrorr   r   )
r  r  r  r  r  r   r  
schema_dctschemar  r/   r/   r0   kv_cache_scales_loader7  s0   

&r	  c                 C   s   ||k rdS t | || S Nr   )min)r  weight_start
weight_endr/   r/   r0   get_actual_shard_sizeb  s   r  c                 C   s8   |dkrd S |dksJ d| |  |||  d S )Nr   z#Length should be positive, but got )r  zero_)r  r  startlengthr/   r/   r0   reset_param_data_if_neededi  s
   r  Tc                 C   sn   t ||||}|r!|dkr||||}n
t| |||}t| ||| ||  | |||} | |fS r
  )r  rT  r  rm   
zeros_liker  )r  r  param_data_startr  r  r  narrow_weightactual_shard_sizer/   r/   r0   r  s  s"   	r  )Nr2   r   )NNr   )F)r  r  r  )T)__doc__r|  concurrent.futuresrt  r  r   rA   r~  r   loggingr)   tempfiler   typingr   r   r   r   r   r   r	   r
   r   rE   huggingface_hub.constantsr,   r<  r.  rp   rt   rm   r   r   r   pydanticr   r   r   r   	tqdm.autor   sglang.srt.configs.load_configr   sglang.srt.configs.model_configr   sglang.srt.distributedr   r   sglang.srt.layers.dp_attentionr   sglang.srt.layers.quantizationr   r   "sglang.srt.layers.quantization.fp8r   -sglang.srt.layers.quantization.modelopt_quantr   r   ,sglang.srt.model_loader.ci_weight_validationr   r   sglang.srt.utilsr    r!   r"   r#   r$   sglang.utilsr%   fastsafetensorsr&   r'   r.   r   	getLoggerr   r   r1   
gettempdirr<   rs  rM   r\   r   r   r   r   r   r   boolr   r  r  r  r  r#  r*  r-  r  rG  rS  rh  rz  r  r  r  r  r  r  r  r  r  LoaderFunctionr  r  r  r  nnModuler  r  r  r  r  r	  r  r  r  r/   r/   r/   r0   <module>   s  ,


(
l4
z
Z
'

&
3

:
'
:





"



 R5
+