o پi@sddlZddlZddlZddlZddlmZmZmZddlm Z ddl m Z m Z m Z mZmZddlZddlmZddlmZddlmZddlmZdd lmZmZmZdd lmZmZm Z m!Z!m"Z"dd l#m$Z$e%e&Z'Gd d d eZ(Gddde)eZ*dede+fddZ,dede-fddZ.dede-fddZ/dede-fddZ0GdddZ1ej2ej2ej3ej3ej4dZ5dedee)ej6fdej6fddZ7d>d!e e)d"e+fd#d$Z8gd%Z9ej:;Zd!e e)fd(d)Z?d!e e)fd*d+Z@d!e e)fd,d-ZAd!e e)fd.d/ZBd!e e)fd0d1ZCd!e e)fd2d3ZDd?d5eEd6eEdeEfd7d8ZFd!e e)fd9d:ZGd!e e)d;efdZd?d@ZdNdAdBZdNdCdDZ dEdFZ!dee"efdGdHZ#deee$ffdIdJZ%dNdKdLZ&dS)O ModelConfigTNz{}rFopenai model_pathtrust_remote_coderevisioncontext_lengthmodel_override_args is_embeddingenable_multimodaldtype quantizationoverride_config_fileis_draft_model model_implsampling_defaultsquantize_and_serveis_multi_layer_eagle encoder_only language_onlydisable_hybrid_swa_memoryr)cCsd||_||_| |_| |_| |_| |_||_||_||_| | t ||_ i}| r7| r7| |d<t|jf|||j d||_t|j|_t|jf||d||_|durygd}|jjd|vrwd}td|jjdnd }|t|jd d|_||_t|jj||_|ot |jj|_!|ot"|jj|_#|ot$|jj|_%|ot&|jj|_&|ot'|jd |_(|ot'|jd |_)|ot*|jj|_*t+|jj|_,t-|jj|_-t.|j||_/|0||1|2|3|4|5|6|_7t|jd dpt|jdd|_8||j_9||j_:t|jdd|_;|j;p.t|jdd|_d|_t j |_ |jj |_ |jj|_|jj|_|jj|_nd |jj vrad!|_t j |_ |jj |_ |jj|_|jj|_|jj|_nd"|jj vsod#|jj vr|jj|_t j |_ |jj |_ |jj|_|jj|_|jj|_dt|j|j|_|jjr|jjdd}|jjd}t|t|}|j|||_nPd$|jj vsd%|jj vsd&|jj vrt|dddur|jj|jj|_t|jdr|jjdurt|jd|jnd'|jj vr|jjd(k|_t j|_ |jj|_t|jd)d|_|jjd*vr(t|jj d+d|_|jdur2|j|_|jj|_|jj!|_!|j!|_"d |jj vrM|j!d,|_"d-|jj vret|jd.d}t#|j!t#||_"t|jd/d|_$|jj%|_%dS)0Nhead_dim v_head_dim swa_head_dimswa_v_head_dimDeepseekV2ForCausalLMr+r*r,rr/rrDotsVLMForCausalLMr-r.MistralLarge3ForCausalLMEagleKimiK25ForConditionalGenerationr rope_typetypedefaultmscale_all_dimFfactorMiniCPM3ForCausalLMDeepseekVL2ForCausalLMuse_mlaTKimiVLForConditionalGenerationKimiLinearForCausalLMHrr MistralModelMixtralForCausalLMMistralForCausalLMBaichuanForCausalLMinum_key_value_heads)dbrxmpt kv_n_headsIQuestLoopCoderForCausalLMloop_numr)&r2rb hidden_sizenum_attention_headsrrrrrar1rrattention_arch kv_lora_rankqk_nope_head_dimqk_rope_head_dimr4r8r7mathsqrtscaling rope_scalingryarn_get_mscalefloatrssetattr use_alibirrrf attn_configrnum_attention_layersintr vocab_size)rrrrscaling_factormscalerrrrr|s                                               z ModelConfig._derive_model_shapescCs|jSr6)rrrrrget_total_num_attention_heads)sz)ModelConfig.get_total_num_attention_headscCs|j}td||S)Nr)rmax)rtensor_parallel_sizetotal_num_attention_headsrrrget_num_attention_heads,sz#ModelConfig.get_num_attention_headscsgd}jj|votjdd}|stjddrdSjjdvr2djjvr.jjdSjjSjjdvrBtjjdjjSjjd vrmfd d jjD}t|d kr]td t|dkrgt dt t |Sgd}|D]}tj|d}|dur|SqsjjS)z%Returns the total number of KV heads.)falcon RefinedWebRefinedWebModelnew_decoder_architectureF multi_queryr)rr)r)z nemotron-nascs$h|]}|jjsjj|jjqSr) attentionno_oprarn_heads_in_group).0blockrrr Os z5ModelConfig.get_total_num_kv_heads..rz%Couldn't determine number of kv headszCVariable GQA (VGQA) is not yet supported for nemotron-nas in sglang) n_head_kv num_kv_headsrmulti_query_group_numnum_attention_groupsN) rarfr2rbrr block_configslen RuntimeErrorrnextiter)rfalcon_model_typesnew_decoder_arch_falconnkvh attributesattrrrrrget_total_num_kv_heads1sH            z"ModelConfig.get_total_num_kv_headscCs|}td||S)z'Returns the number of KV heads per GPU.r)rrrrtotal_num_kv_headsrrrget_num_kv_headspszModelConfig.get_num_kv_headscCsTt|jdr|jj}td||St|jdr%|jjd}td||S||S)z+Similar to get_num_kv_heads(), but for SWA.swa_num_key_value_headsrattention_other_settingr)rsrbrrrrrrrrrget_swa_num_kv_headsys   z ModelConfig.get_swa_num_kv_headsc s\tjdd}|durt|ts|}|dur*d|vr*d|i}|r*|||dur5tjdd}|dur,tj j }|st j rTddlm}m}|nddl}ddlm}m}|zTd} |jjs|tfd d d d d d} | s||WSt j r|j djd} n |j dj|jjd} t|  } t| } Wdn1swY| }W|S|jjytdj dY|S|jjyt dY|St!y} zt dj | WYd} ~ |Sd} ~ wwtj tj"j dr,tj"j d} t|  } t| } Wdn 1s"wY| }|S)Nquantization_config quant_methodrGcompression_configr)HubApimodel_file_download)HfApihf_hub_downloadFcsjdS)Nhf_quant_config.json) file_existsr?rhf_apirrrsz4ModelConfig._parse_quant_hf_config..r?g@) max_retry initial_delay max_delayr)model_id file_pathrA)repo_idfilenamerAlocal_files_onlyz,hf_quant_config.json not found in cache for z0 (offline mode, normal for non-quantized models)zCheck if the model is already quantized based on config files.rNrTr)has_hf_quant_config)r2rasglang.srt.utilsrWr?)rrWrrr_is_already_quantizeds   z!ModelConfig._is_already_quantizedcCsb|jdkrdS|jdkrdS|jdkr/|}|r-|dd}d|vr'dSd|vr-dSdSdS) zBExtract ModelOpt quantization type from unified quantization flag.rTfp8rRnvfp4modeloptrfp4)rGrIrlower)rrBrrrr_get_modelopt_quant_type s   z$ModelConfig._get_modelopt_quant_typecCs(t|jdd}|durt|jdd}|S)Nrisliding_window)r2rb)rrirrrrh!sz$ModelConfig._get_sliding_window_sizecCsB|jsdSgd}|j|v}|stddt|dtd)z*Validate quantize_and_serve configuration.N)r\rTrRzLquantize_and_serve requires ModelOpt quantization (set with --quantization {z, z})zquantize_and_serve functionality is currently disabled due to compatibility issues. Please use the separate quantize-then-deploy workflow instead. Step 1: Quantize and export model. Step 2: Deploy the exported model.)rLrGrrAsortedNotImplementedError)r_MODELOPT_QUANTIZATION_METHODSmodelopt_quantization_specifiedrrrr\'s z/ModelConfig._validate_quantize_and_serve_configcCs^gt}gd}gd}dgdgdgddgddgd}|jdur&|j|_g}|}|}|p3|}|dur=||dd|D}t|d krNtd |rT|d nd} | dur| d |jsbd n|j} t D]\} } | | |j} | r| } | |_nql|jdur| |_nC|j| kr|j|vo| ||jv}|rt d|jd| dn"|j rt d| d|jd| | |_n td| d|jd| dddk|_d dlm}|js|jrt d|jdur)|j|vrtd|jd|dtr|j|vrt|jd|j|vr+|jdkr ts-t d|jdSdSdSdS) N) awqgptqrZcompressed_tensorscompressed-tensors fbgemm_fp8w8a8_fp8 petit_nvfp4quarkmxfp4z auto-roundquark_int4fp8_moe)rZmarlinrTrRgptq_marlin_24 gptq_marlin awq_marlinrjrhri experts_int8 w8a8_int8rk moe_wna16qoqrOrlrmrJr\rirh)rTrRrlrurkcSsg|]}|dur|qSr6r)ritemrrr ~sz4ModelConfig._verify_quantization..rz;Config list contains configs from 2 methods, must be only 1rrr]z"Using CLI-specified quantization (z3) which is compatible with HF config quant_method (z).zDraft model quantization (z() differs from main model quantization (z.). Using draft model's detected quantization: z3Quantization method specified in the model config (zS) does not match the quantization method specified in the `quantization` argument ( scale_fmtue8m0)deep_gemm_wrapperzuDeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell.zUnknown quantization method: z. Must be one of rz1 quantization is currently not supported in ROCm.rnz^%s quantization is not fully optimized yet. The speed can be slower than non-quantized models.)r rGr_rIrLappendrrritemsoverride_quantization_methodrdrerIuse_scale_ue8m0sglang.srt.layersr|DEEPGEMM_SCALE_UE8M0rrr)rsupported_quantizationrocm_supported_quantizationoptimized_quantization_methodscompatible_quantization_methodscfg_listramodelslim_config quant_configrBr_methodquantization_override is_compatibler|rrrr~Es                z ModelConfig._verify_quantizationcCsNt|jdr#t|j}|sdS||jjd<d|jjvr%d|jjd<dSdSdS)Ndual_chunk_attention_configsparse_attention_configsparse_attention_enabledT)rsrarr?r)rsparse_attn_configrrrrs    z/ModelConfig._verify_dual_chunk_attention_configc Csddl}ddlm}t|dd}|durdSt|jdd}d|jvp/|duo/t|dddk}|}||}|d}||krU|rStd |d |jd |jj d dS|sgt d |d |jj ddSdS)Nr)version __version__rVzglm-4.6vrfglm4v_moe_visionz 5.0.0dev0zTransformers version z is not supported for model z or model type z*. Please upgrade transformers to >= 5.0.0.z is used for model type z. If you experience issues related to RoPE parameters, they may be due to incompatibilities between Transformers >=5.0.0 and some models. You can try downgrading to transformers==4.57.1 as a workaround.) r" packagingrr2rar?r_parserrfrdr) rr"rtf_version_strrV is_glm_46vmoe needs_tf_v5 tf_versionrequired_versionrrrrs4    z(ModelConfig._verify_transformers_versioncCsxt|jdd}|durt|tr|hnt|}|durt}|jr:t|jdd}|r:t|tr2|hnt|}||B}|S)N eos_token_id)r2rar-rsetrc)reos_idsgeneration_eos_idsrrrrs z ModelConfig._get_hf_eos_token_idcsD|jdkriS|jduriS|jgd}fdd|D}|S)aG Get default sampling parameters from the model's generation config. This method returns non-default sampling parameters from the model's generation_config.json when sampling_defaults is set to "model". Returns: A dictionary containing the non-default sampling parameters. modelN)repetition_penalty temperaturetop_ktop_pmin_pcs&i|]}|dur||qSr6rU)rpr3rr 6s z;ModelConfig.get_default_sampling_params..)rKrcr/)ravailable_paramsdefault_sampling_paramsrr3rget_default_sampling_paramss    z'ModelConfig.get_default_sampling_paramscCslddlm}ddlm}||jr2td||j}||jr4|jdgd|j|_| |_dSdSdS)z Pull the model config files to a temporary directory in case of remote. Args: model: The model name or path. r)create_remote_connector) is_remote_urlz$Pulling model configs from remote...z *config.json) allow_patternN) sglang.srt.connectorrrXrr?rdre pull_files model_weights get_local_dir)rrrclientrrrr]<s     z3ModelConfig._maybe_pull_model_tokenizer_from_remote)NNF)r)N)'rrrr r$strboolrrr r staticmethodrrrgr}r{r|rrrrrrIrLr.r0rYr`rhr\r~rrr rrrr]rrrrr=Us       'C"!? _    # r=)halffloat16rfloat32bfloat16rFcCsnt|tr|ddp|dd}|dd}n t|dd}t|dd}t|tr/t|d}|dur6tj}t|tr||}|dkrl|tjkri| dre|dkrTd}n|d}t d|d tj }n'tj }n#|}n |tvrwtd |t|}nt|tjr|}ntd |||kr|tjkrt d || |S|tjkrt d || |St d |||S)NrF torch_dtyperfr]rgemmaz For Gemma zs, we downcast float32 to bfloat16 instead of float16 by default. Please specify `dtype` if you want to use float16.zUnknown dtype: zUpcasting %s to %s.zDowncasting %s to %s.zCasting %s to %s.)r-r.rr2r_STR_DTYPE_TO_TORCH_DTYPEtorchrr_ startswithrdrerrrrFr)r(rF config_dtyperf gemma_versionrrrrrz_sP             rzFmodel_architecturesrDcCsd|vs@d|vs@d|vs@d|vs@d|vs@d|vs@d|vs@d|vs@d |vs@d |vs@d |vs@d |vs@d |vs@d|vs@d|vs@d|vrBdS| S)NLlamaEmbeddingModelrLlamaForSequenceClassification0LlamaForSequenceClassificationWithNormal_WeightsInternLM2ForRewardModelQwen2ForRewardModelQwen3ForRewardModelQwen2ForSequenceClassificationQwen3ForSequenceClassification CLIPModel BertModel ContrieverBertForSequenceClassificationXLMRobertaModel#XLMRobertaForSequenceClassificationGemma2ForSequenceClassificationFr)rrDrrrrjs$rj)3rr&Ernie4_5_VLMoeForConditionalGenerationrRGemma3nForConditionalGenerationGlm4vForConditionalGeneration Glm4vMoeForConditionalGenerationrGlmAsrForConditionalGenerationGrok1VForCausalLMGrok1AForCausalLMLlavaLlamaForCausalLMrSLlavaMistralForCausalLMLlavaQwenForCausalLMLlavaForConditionalGenerationLlavaVidForCausalLM"LightOnOCRForConditionalGenerationMiniCPMOMiniCPMV Mistral3ForConditionalGenerationMultiModalityCausalLMMllamaForConditionalGenerationNemotronH_Nano_VL_V2r."Qwen2AudioForConditionalGenerationQwen2VLForConditionalGeneration"Qwen2_5_VLForConditionalGenerationQwen3VLForConditionalGeneration"Qwen3VLMoeForConditionalGenerationrr$Qwen3OmniMoeForConditionalGenerationrInternVLChatModel InternS1ForConditionalGeneration#InternS1ProForConditionalGenerationPhi4MMForCausalLMrTPOINTSV15ChatModelrDotsOCRForCausalLMSarashina2VisionForCausalLMNVILAForConditionalGeneration!NVILALiteForConditionalGenerationDeepseekOCRForCausalLMJetVLMForConditionalGeneration#PaddleOCRVLForConditionalGenerationMiDashengLMModelStepVLForConditionalGenerationrcstfddtDr dSdS)Nc3|]}|vVqdSr6rrmulti_model_archrrr s  z&is_multimodal_model..TF)anymultimodal_model_archsrrrrrls rlcCdSNFrrrrrrnrncCrrrrrrrrprrpcCrrrrrrrrrrrrcCd|vS)Nrrrrrrrw rwcCr)NrSrrrrrryrrycs&gdtfdd|DrdSdS)z=Check if chunked prefill is supported for a MultiModal model.)rrrrrc3rr6rr unsupportedrrrz:is_multimodal_chunked_prefill_supported..FTrrrrrrvsrvrscalercCs"|dkrdSd|t|dS)Nrr$g?)rlog)r rrrrr"srcshdtfdd|DS)N>rrGptOssForCausalLMrrrSc3rr6r)rarchhybrid_swa_archsrrr2rz&is_hybrid_swa_model..rrrr rr(srrbcsH|jd|vrddtD}ddtD}||fSd|vr=t|dd}ddt|D}ddt|D}||fSd |vrat|d dfd dtD}fd dtD}||fSd |vrndg}g}||fSd|vr|j}fddt|D}fddt|D}||fSd|vrdg}g}||fSd}d}||fS)NrScSs g|] }|dddkr|qSrrrrirrrry;z(get_hybrid_layer_ids..cSs g|] }|dddkr|qSrrrrrrry>rr  layer_typescSg|] \}}|dkr|qSsliding_attentionrrrxrrrryCcSrfull_attentionrrrrrryFrrhybrid_layer_patterncg|] }|dkr|qS)rrrrrrryKrcr)rrrrrrryNrrrrc$g|]\}}|dkr|kr|qSrrrrrrryV cr rrrr!rrry[r"r)rranger2 enumerater)rrbrrrr)rrrr5s^(        r)F)rr)Ir^loggingrr2enumrrrpathlibrtypingrrrr r rr"r sglang.srt.environr sglang.srt.layers.quantizationr sglang.srt.server_argsrrXrrr&sglang.srt.utils.hf_transformers_utilsrrrrr sglang.utilsr getLoggerrrdrrr rr4rr8r:r<r=rrrrrFrzrjrSGLANG_EXTERNAL_MM_MODEL_ARCHrexternal_mm_model_archr}rlrnrprrrwryrvrrrrrrrrs|           @6