o پi*@s,UdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z mZmZmZmZmZmZddlmZddlmZddlmZddlmZdd lmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;mZ>m?Z?m@Z@mAZAdd lBmCZCdd lDmEZEeFeGZHdZIhdZJgdZKgdZLgeLdZMgdZNgdZOgdZPgdZQgdZRgdZSddgZTddgZUdZVgdZWddgZXd gZYgd!ZZgd"Z[gd#Z\gd$Z]gd%Z^gd&Z_dd'gZ`d(d)Zad*d+Zbd,d-Zcd.d/Zdd0d1Zed2d3Zfd4d5Zgd6d7Zhd8d9Zid:d;Zjdd?Zld@dAZmejnGdBdCdCZodapdDeqdE<d_dGdHZrerZsd`dJdKZtdadNdOZudPZvdQZwejnGdRdSdSZxGdTdUdUejyZzdbdXdYZ{GdZd[d[ejyZ|dcd]d^Z}dS)dzThe arguments of the server.) annotationsN)AnyCallableDictListLiteralOptionalUnion) ConnectorType)envs)FunctionCallParser) CHUNK_SIZELoRARef)ReasoningParser) LORA_TARGET_ALL_MODULESSUPPORTED_LORA_TARGET_MODULEScheck_pkg_version_at_leastconfigure_ipv6cpu_has_amx_supportget_bool_env_var get_deviceget_device_memory_capacityget_device_name get_device_sm get_free_portget_int_env_varget_quantization_configis_blackwell_supportedis_cudais_flashinfer_availableis_hipis_hopper_with_cuda_12_3is_no_spec_infer_or_topk_oneis_npu is_remote_urlis_sm90_supportedis_sm100_supportedis_sm120_supportedis_triton_kernels_availableis_valid_ipv6_addressjson_list_type nullable_strparse_connector_type torch_releasewait_port_availablexpu_has_xmx_support)check_gguf_file)is_in_ci>ascendpytorch flashinfer)autopt safetensorsnpcachedummy sharded_stategguf bitsandbyteslayeredflash_rlremoteremote_instancefastsafetensorsprivate)awqfp8mxfp8gptqmarlin gptq_marlin awq_marlinr>r=modelopt modelopt_fp8 modelopt_fp4 petit_nvfp4 w8a8_int8w8a8_fp8 moe_wna16qoqw4afp8mxfp4z auto-roundcompressed-tensors modelslimquark_int4fp8_moeunquant)triton torch_nativeflex_attentionnsa cutlass_mlafa3fa4r6flashmla trtllm_mla trtllm_mhadual_chunk_flash_attnaiterwave intel_amxr4 intel_xpu)rZcsgmvr4r[)mooncakenixlr4fakemori)zmq_to_schedulerzmq_to_tokenizerrj)xgrammaroutlines llguidancenone)r6r_rZr_rZ in-seq-splitround-robin-splitlru)flashmla_sparse flashmla_kv flashmla_autor_tilelangretrtllmlfufsdp) r7 deep_gemmrZ triton_kernelflashinfer_trtllmflashinfer_cutlassflashinfer_mxfp4flashinfer_cutedslcutlass)rsdeepeprjrm ascend_fuseepr6)r7r~rflashinfer_deepgemmrrZre)r7flashinfer_cudnnrr)float32bfloat16float16)r7 no_buffer extra_bufferr6cCt|dSN)LOAD_FORMAT_CHOICESextendchoicesr3r3J/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/server_args.pyadd_load_format_choicesrcCrr)QUANTIZATION_CHOICESrrr3r3radd_quantization_method_choicesrrcCrr)ATTENTION_BACKEND_CHOICESrrr3r3radd_attention_backend_choicesrrcCrr)DISAGG_TRANSFER_BACKEND_CHOICESrrr3r3r#add_disagg_transfer_backend_choicesrrcCrr)GRAMMAR_BACKEND_CHOICESrrr3r3radd_grammar_backend_choicesrrcCrr)MOE_RUNNER_BACKEND_CHOICESrrr3r3radd_moe_runner_backend_choicesrrcCrr)FP8_GEMM_RUNNER_BACKEND_CHOICESrrr3r3r#add_fp8_gemm_runner_backend_choicesrrcCrr)FP4_GEMM_RUNNER_BACKEND_CHOICESrrr3r3r#add_fp4_gemm_runner_backend_choicesrrcCrr)'DETERMINISTIC_ATTENTION_BACKEND_CHOICESrrr3r3r+add_deterministic_attention_backend_choicesrrcCrr)/RADIX_SUPPORTED_DETERMINISTIC_ATTENTION_BACKENDrrr3r3r;add_radix_supported_deterministic_attention_backend_choicesrrcCrr)RADIX_EVICTION_POLICY_CHOICESrrr3r3r!add_radix_eviction_policy_choicesrrcCrr)RL_ON_POLICY_TARGET_CHOICESrrr3r3radd_rl_on_policy_target_choices rrcCrr)MAMBA_SSM_DTYPE_CHOICESrrr3r3radd_mamba_ssm_dtype_choicesrrc@seZdZUdZded<dZded<dZded<d Zd ed <d Zd ed<dZ ded<dZ ded<d Z d ed<dZ ded<d Z d ed<dZded<dZded<dZded<dZded<dZd ed<dZded<d Zd ed <d Zd ed!<dZded"<dZded#<d Zd ed$<dZded%<dZded&<dZded'<dZded(<d Zd ed)<dZd*ed+<dZded,<dZ ded-<dZ!ded.<d Z"d ed/<dZ#ded0<dZ$d1ed2<dZ%ded3<dZ&ded4<dZ'ded5<dZ(ded6<d Z)d ed7<d8Z*d ed9<dZ+ded:<d;Z,ded<<d Z-d ed=<d Z.d ed><d Z/d ed?<d@Z0d edA<dBZ1dCedD<dZ2dedE<dFZ3dCedG<d Z4d edH<dIZ5dedJ<d Z6d edK<dLZ7d edM<dZ8d1edN<dZ9dOedP<dZ:dOedQ<dZ;dedR<d ZdedU<dVZ?d edW<d Z@d edX<d ZAd edY<dZBdedZ<dZCded[<d ZDd ed\<d]ZEdCed^<dZFd1ed_<dZGded`<dZHdeda<dZIdedb<dVZJd edc<d ZKd edd<d ZLd ede<dZMdfedg<dhZNdedi<dZOdedj<d ZPd edk<dlZQd edm<dnZRdedo<dZSdpedq<eTjUdrdsdtZVduedv<dZWdedw<d ZXd edx<d ZYd edy<d ZZd edz<d{Z[ded|<dZ\dped}<dZ]d~ed<dZ^dOed<dZ_dOed<dZ`dOed<d Zad ed<dZbdped<dZcdped<dZddCed<dZed ed<d Zfd ed<dZgded<d Zhd ed<dZided<d Zjd ed<dZkded<dZlded<dZmded<dZnded<dZoded<dZpded<dZqded<dZrded<dZsded<d Ztd ed<dZuded<dZvded<dZwded<dZxded<d Zyd ed<dZzded<d Z{d ed<d Z|d ed<dZ}ded<d Z~d ed<dVZd ed<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dZd ed<dIZded<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dZded<d Zd ed<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dZded<dBZdCed<dBZdCed<dZded<dZded<dZded<dZded<dZded<dZded<d Zd ed<dZd ed<d Zd ed<d@Zd ed<dZded<dZd ed<dZd ed<d Zd ed<d Zd ed<dZded<dZded<dZded<d Zd ed<dZded<dVZd ed<dZded<dZded<d Zd ed<dZded<dZd ed<dZded<dBZdCed<dZded<dZded<d Zd ed<dZded<dZded<dZded<dZded<dZded<dZded<dZdCed<dZded<dZd ed<d Zd ed <d ZdCed <dVZd ed <d Zded<dZded<dZded<d Zd ed<dZded<dZded<dZded<dZded<d Zd ed<dZded<dZded<dZded<dZded<dZded<dZded<dZded <dZded!<d Zd ed"<dZded#<d$Zd ed%<dZd ed&<d'Zded(<d)Zd ed*<dVZd ed+<d,Zd ed-<d Zd ed.<d Zd ed/<d0Zded1<dZd2ed3<d Zd ed4<dZded5<dZd6ed7<d Zd ed8<d Zd ed9<d Zd ed:<d Zd ed;<d Zd ed<<d Zd ed=<d Zd ed><d Zd ed?<d Zd ed@<d Zd edA<d Zd edB<d Zd edC<d Zd edD<d Zd edE<d Zd edF<d Zd edG<d Zd edH<d Zd edI<d Zd edJ<d Zd edK<dLZdCedM<d Zd edN<d Zd edO<d Zd edP<d$Zd edQ<dZdedR<dZd6edS<dTZdedU<dZdedV<d Z d edW<d Z d edX<d Z d edY<dZ d edZ<dZ ded[<d Zd ed\<d Zd ed]<d Zd ed^<d Zd ed_<d Zd ed`<d Zd eda<d Zd edb<d Zd edc<d Zd edd<d Zd ede<d Zd edf<d Zd edg<d Zd edh<d Zd edi<d Zd edj<dZd6edk<d Zd edl<dZdedm<d Z d edn<d Z!d edo<dpZ"dedq<d Z#d edr<d Z$d eds<d Z%d edt<d$Z&d edu<dvZ'dCedw<dZ(dedx<dZ)d6edy<dZ*dedz<d Z+d ed{<d|Z,d}ed~<dZ-ded<dZ.d ed<dZ/ded<dZ0ded<d Z1ded<dZ2ded<d Z3d ed<dZ4d ed<d Z5d ed<d Z6d ed<d Z7d ed<e8dVZ9ded<eTjUe:dtZ;dued<dZded<dZ?ded<dZ@d6ed<dZAded<d ZBd ed<d ZCd ed<dZDded<dZEd ed<d$ZFd ed<dZGdCed<d ZHd ed<d ZId ed<d ZJd ed<dZKded<dZLded<dZMded<dZNded<dZOded<ddZPddZQddZRddZSddZTddZUddZVddZWddZXddZYddZZd1dd„Z[d2dÐdĄZ\dŐdƄZ] ǐ d3d4d̐d̈́Z^dΐdτZ_dАdфZ`dҐdӄZadԐdՄZbd֐dׄZcdؐdلZddڐdۄZedܐd݄Zfdސd߄ZgddZhddZiddZjddZkddZlddZmddZnddZoddZpddZqddZrd5ddZsddZtddZuddZvddZwddZxddZyddZze{d6d d Z|e}d7d dZ~ddZddZddZddZddZd8ddZed9ddZddZdd Zd!d"Zd:d%d&Zd;d)d*Zd+d,Zd-d.Zd/d0ZdS(< ServerArgsa8 The arguments of the server. NOTE: When you add new arguments, please make sure the order in this class definition the same as the order in the the function `ServerArgs.add_cli_args`. Please follow the existing style to group the new arguments into related groups or create new groups. str model_pathN Optional[str]tokenizer_pathr7tokenizer_modeinttokenizer_worker_numFboolskip_tokenizer_init load_formatz{}model_loader_extra_configtrust_remote_code Optional[int]context_length is_embeddingzOptional[bool]enable_multimodalrevision model_impl 127.0.0.1hosti0uportfastapi_root_path grpc_modeskip_server_warmupwarmups nccl_port+checkpoint_engine_wait_weights_before_readydtype quantizationquantization_param_pathkv_cache_dtypeenable_fp32_lm_headzOptional[Union[str, Dict]]modelopt_quant modelopt_checkpoint_restore_pathmodelopt_checkpoint_save_pathmodelopt_export_pathquantize_and_serverl_quant_profilezOptional[float]mem_fraction_staticmax_running_requestsmax_queued_requestsmax_total_tokenschunked_prefill_sizeenable_dynamic_chunking@max_prefill_tokensprefill_max_requestsfcfsschedule_policyenable_priority_schedulingabort_on_priority_when_disabled"schedule_low_priority_values_first (priority_scheduling_preemption_threshold?floatschedule_conservativeness page_size皙?swa_full_tokens_ratiodisable_hybrid_swa_memoryrvradix_eviction_policyenable_prefill_delayer prefill_delayer_max_delay_passes)prefill_delayer_token_usage_low_watermarkzOptional[List[float]]&prefill_delayer_forward_passes_buckets$prefill_delayer_wait_seconds_bucketsdevicetp_sizepp_sizepp_max_micro_batch_sizerpp_async_batch_depthstream_interval stream_output random_seed#constrained_json_whitespace_pattern'constrained_json_disable_any_whitespace,watchdog_timeoutsoft_watchdog_timeout dist_timeout download_dirmodel_checksum base_gpu_id gpu_id_step sleep_on_idlezOptional[Callable]custom_sigquit_handlerinfo log_levellog_level_http log_requestslog_requests_leveltextlog_requests_formatzOptional[List[str]]log_requests_targetcCsttSr)list+DEFAULT_UVICORN_ACCESS_LOG_EXCLUDE_PREFIXESr3r3r3ryszServerArgs.)default_factory List[str]#uvicorn_access_log_exclude_prefixescrash_dump_foldershow_time_costenable_metrics!enable_metrics_for_all_schedulerszx-custom-labels&tokenizer_metrics_custom_labels_header'tokenizer_metrics_allowed_custom_labelszOptional[Dict[str, str]]extra_metric_labelsbucket_time_to_first_tokenbucket_inter_token_latencybucket_e2e_request_latencycollect_tokens_histogramprompt_tokens_bucketsgeneration_tokens_bucketsggc_warning_threshold_secs(decode_log_interval!enable_request_time_stats_loggingkv_events_config enable_tracelocalhost:4317otlp_traces_endpointexport_metrics_to_fileexport_metrics_to_file_dirapi_key admin_api_keyserved_model_namedefaultweight_version chat_templatehf_chat_template_namecompletion_templatesglang_storagefile_storage_pathenable_cache_reportreasoning_parsertool_call_parser tool_servermodelsampling_defaultsdp_sizeload_balance_method attn_cp_size moe_dp_sizedist_init_addrnnodes node_rankjson_model_override_argspreferred_sampling_params enable_loraenable_lora_overlap_loading max_lora_rankz$Optional[Union[set[str], List[str]]]lora_target_moduleszOOptional[Union[dict[str, str], List[dict[str, str]], List[str], List[LoRARef]]] lora_pathsmax_loaded_lorasmax_loras_per_batchlora_eviction_policyri lora_backendmax_lora_chunk_sizeattention_backenddecode_attention_backendprefill_attention_backendsampling_backendgrammar_backendmm_attention_backendfp8_gemm_runner_backendrfp4_gemm_runner_backendnsa_prefill_backendnsa_decode_backenddisable_flashinfer_autotunerZ mamba_backendspeculative_algorithmspeculative_draft_model_path speculative_draft_model_revisionspeculative_draft_load_formatspeculative_num_stepsspeculative_eagle_topkspeculative_num_draft_tokens#speculative_accept_threshold_single speculative_accept_threshold_accspeculative_token_mapprefillspeculative_attention_mode#speculative_draft_attention_backendspeculative_moe_runner_backendspeculative_moe_a2a_backend$speculative_draft_model_quantization'speculative_ngram_min_match_window_size 'speculative_ngram_max_match_window_size!speculative_ngram_min_bfs_breadth!speculative_ngram_max_bfs_breadthBFSzLiteral['BFS', 'PROB']speculative_ngram_match_typespeculative_ngram_branch_lengthispeculative_ngram_capacityenable_multi_layer_eagleep_sizerszLLiteral['none', 'deepep', 'mooncake', 'mori', 'ascend_fuseep', 'flashinfer']moe_a2a_backendmoe_runner_backendzLiteral['default', 'bf16']flashinfer_mxfp4_moe_precision"enable_flashinfer_allreduce_fusionz(Literal['auto', 'normal', 'low_latency'] deepep_modeep_num_redundant_expertsz.Optional[Literal['static', 'dynamic', 'fake']]ep_dispatch_algorithmtrivialinit_expert_location enable_eplbeplb_algorithmeplb_rebalance_num_iterationseplb_rebalance_layers_per_chunk*eplb_min_rebalancing_utilization_thresholdzAOptional[Literal['stat', 'stat_approx', 'per_pass', 'per_token']]!expert_distribution_recorder_mode(expert_distribution_recorder_buffer_size"enable_expert_distribution_metrics deepep_configmoe_dense_tp_sizezLiteral[None, 'mooncake']elastic_ep_backendmooncake_ib_devicemax_mamba_cache_sizemamba_ssm_dtypeg?mamba_full_memory_ratiomamba_scheduler_strategymamba_track_intervalenable_hierarchical_cacheg@ hicache_ratio hicache_size write_throughhicache_write_policykernelhicache_io_backend layer_firsthicache_mem_layoutdisable_hicache_numa_detecthicache_storage_backend best_efforthicache_storage_prefetch_policy$hicache_storage_backend_extra_config*hierarchical_sparse_attention_extra_configenable_lmcachekt_weight_path kt_method kt_cpuinferkt_threadpool_countkt_num_gpu_experts!kt_max_deferred_experts_per_tokendllm_algorithmdllm_algorithm_configenable_double_sparsityds_channel_config_path ds_heavy_channel_numds_heavy_token_numqkds_heavy_channel_typeds_sparse_decode_thresholdcpu_offload_gboffload_group_sizeoffload_num_in_groupoffload_prefetch_stepcpu offload_modezOptional[Union[int]]multi_item_scoring_delimiterdisable_radix_cachecuda_graph_max_bsOptional[List[int]] cuda_graph_bsdisable_cuda_graphdisable_cuda_graph_paddingenable_profile_cuda_graphenable_cudagraph_gcenable_layerwise_nvtx_markerenable_nccl_nvlsenable_symm_mem,disable_flashinfer_cutlass_moe_fp4_allgatherenable_tokenizer_batch_encodedisable_tokenizer_batch_decodedisable_outlines_disk_cachedisable_custom_all_reduceenable_mscclppenable_torch_symm_memdisable_overlap_scheduleenable_mixed_chunkenable_dp_attentionenable_dp_lm_headenable_two_batch_overlapenable_single_batch_overlapgQ? tbo_token_distribution_thresholdenable_torch_compileenable_piecewise_cuda_graphenable_torch_compile_debug_modetorch_compile_max_bspiecewise_cuda_graph_max_tokenspiecewise_cuda_graph_tokenseagerpiecewise_cuda_graph_compilertorchao_configenable_nan_detectionenable_p2p_checktriton_attention_reduce_in_fp32triton_attention_num_kv_splits triton_attention_split_tile_sizenum_continuous_decode_stepsdelete_ckpt_after_loadingenable_memory_saverenable_weights_cpu_backupenable_draft_weights_cpu_backupallow_auto_truncateenable_custom_logit_processorflashinfer_mla_disable_raggeddisable_shared_experts_fusiondisable_chunked_prefix_cachedisable_fast_image_processorkeep_mm_feature_on_deviceenable_return_hidden_statesenable_return_routed_expertsscheduler_recv_interval numa_nodeenable_deterministic_inferencerl_on_policy_targetenable_attn_tp_input_scattered#enable_nsa_prefill_context_parallelrunsa_prefill_cp_modeenable_fused_qk_norm_rope&enable_precise_embedding_interpolationenable_dynamic_batch_tokenizer"dynamic_batch_tokenizer_batch_sizegMb`?%dynamic_batch_tokenizer_batch_timeoutdebug_tensor_dump_output_folderdebug_tensor_dump_layersdebug_tensor_dump_input_filedebug_tensor_dump_injectnullz$Literal['null', 'prefill', 'decode']disaggregation_moderjdisaggregation_transfer_backendi&#disaggregation_bootstrap_portdisaggregation_decode_tpdisaggregation_decode_dpdisaggregation_prefill_ppdisaggregation_ib_device,disaggregation_decode_enable_offload_kvcachenum_reserved_decode_tokens&disaggregation_decode_polling_interval encoder_only language_onlyencoder_transfer_backend encoder_urlscustom_weight_loaderweight_loader_disable_mmap.remote_instance_weight_loader_seed_instance_ip8remote_instance_weight_loader_seed_instance_service_port6remote_instance_weight_loader_send_weights_group_portsncclz"Literal['transfer_engine', 'nccl']%remote_instance_weight_loader_backend|t|jd7}|durM|d krMt|d}|jdurh|jdkr^|d7}n |jdkrh|d7}|durvt|||dnd|_ |}|jr|js|||jrtjstjdtddSdSdS)a# Configure GPU memory-dependent settings including chunked_prefill_size, cuda_graph_max_bs, and mem_fraction_static. Here are our heuristics: - Set chunked_prefill_size and cuda_graph_max_bs based on the GPU memory capacity. This is because GPUs with more memory are generally more powerful, we need to use a larger chunked_prefill_size and a larger cuda_graph_max_bs to fully utilize the GPU. - Then set mem_fraction_static based on chunked_prefill_size and cuda_graph_max_bs. GPU memory capacity = model weights + KV cache pool + activations + cuda graph buffers The argument mem_fraction_static is defined as (model weights + KV cache pool) / GPU memory capacity, or equivalently, mem_fraction_static = (GPU memory capacity - activations - cuda graph buffers) / GPU memory capacity. In order to compute mem_fraction_static, we need to estimate the size of activations and cuda graph buffers. The activation memory is proportional to the chunked_prefill_size. The cuda graph memory is proportional to the cuda_graph_max_bs. We use reserved_mem = chunked_prefill_size * 1.5 + cuda_graph_max_bs * 2 to estimate the size of activations and cuda graph buffers in GB. and set mem_fraction_static = (GPU memory capacity - reserved_mem) / GPU memory capacity. The coefficient 1.5 is a heuristic value, in the future, we can do better estimation by looking at the model types, hidden sizes or even do a dummy run. NiPirOiPirrih rrirrg?rri( STANDALONEiNGRAMg)\(?zSymmetric memory is enabled, setting symmetric memory prealloc size to 4GB as default.Use environment variable SGLANG_SYMM_MEM_PREALLOC_GB_SIZE to change the prealloc size.)rrrr _generate_cuda_graph_batch_sizesmaxruse_mla_backendr%_generate_piecewise_cuda_graph_tokensrrrrr@rlenraroundget_model_config is_multimodalradjust_mem_fraction_for_vlmrr SGLANG_SYMM_MEM_PREALLOC_GB_SIZEis_setsetrYrZ)rQrR reserved_mem model_configr3r3rr4s                                   z&ServerArgs._handle_gpu_memory_settingscsjrttdjd}nPjdur3gdttdddttdddttdjdd }n+ttdd dttd d d ttdddttdddttdjdd}fdd|D}|S)z Generate the list of batch sizes for CUDA graph capture based on cuda_graph_max_bs. This integrates the logic from cuda_graph_runner.py. rN)rrrvrOrrrSrOirr r!rr'ArvHcg|] }|jkr|qSr3)r).0bsr[r3r Qsz?ServerArgs._generate_cuda_graph_batch_sizes..)rrrangerra)rQ capture_bsr3r[rr7s.  z+ServerArgs._generate_cuda_graph_batch_sizescs|ttdddttdddttdddttd d d ttd d dttdjdd}fdd|D}|S)z Generate the list of batch sizes for piecewise CUDA graph capture based on piecewise_cuda_graph_max_tokens. rvr0rrSi iri@i@iirirrcrr3)r)rsr[r3rrcszDServerArgs._generate_piecewise_cuda_graph_tokens..)rrr)rQ capture_sizesr3r[rrUs  z0ServerArgs._generate_piecewise_cuda_graph_tokensmajorreturncCs|jdu}|jdu}|s|r|jdkrtd|jdkr3|dkr#dnd|_td|jd|d|jd kr;d|_|jd vsDJd dS) Nr7zWhen specifying --nsa-prefill-backend or --nsa-decode-backend, you should also explicitly set --kv-cache-dtype (e.g., 'fp8_e4m3' or 'bfloat16'). DeepSeek V3.2 defaults to FP8 KV cache which may not be compatible with all backends.rfp8_e4m3rzSetting KV cache dtype to z for DeepSeek DSA on SMz device.bf16)rrzCDeepSeek DSA only supports bf16/bfloat16 or fp8_e4m3 kv_cache_dtype)r]r^rrYrZ)rQruser_set_prefilluser_set_decoder3r3r_set_default_nsa_kv_cache_dtypeis     z*ServerArgs._set_default_nsa_kv_cache_dtypec Cs|jdu}|jdu}|dkr|sd|_|sd|_n|dkr(|s"d|_|s'd|_n |s-d|_|s2d|_td|jd |jd |jd dS) Nrryrxrrwr{r_zSet NSA backends for z KV Cache: prefill=z , decode=.)r]r^rYrZr)rQrrrrr3r3r_set_default_nsa_backendss*  z$ServerArgs._set_default_nsa_backendscCs> ddlm}t|jtjkrdS|j}|jd}|dvr!d|_ |dvr||r|dkrt)|}|d?k} | rd|_ |j+d,krJt r | r d@|_+tdAn?tr t4dBr | r d,|_+tdCn*tr2t4dBr2d4|_+tdDn|jd krJt5rJ|j*durJdE|_+tdF|j+dEkrZ|jd ksZJdGndH|vr|j,d1krd|_6tdIt j70st j7 dtdJ|j8rdK|_9tdLd|_3tdMndN|vr|j,d1krd|_6tdOt j70st j7 dtdJ|j8rdK|_9tdPd|_3tdQn{dR|vrE|j:dSkrE|jdurt&rdT\|_} n%t1rdU\|_} ntrdV\|_} n|j:dWkr dX\|_} ndY\|_} tdZ|jd[| d\|jd]vs+Jd^|jt&rC|j+d,krC|j*d_vrCd.|_+td`n|davrXtdb|dcd|_3n|ddvr|j;durtde|dcd|_3gdf} |j| vsJdg| dh|di|jn|djvrtde|dcd|_3|jdurt<rt&rd7|_nt<rt=dkkrd8|_nd4|_|jdlksJdmtdn|jdo|dpnu|dqvr|j>|drdsng|dtvr-|}|j*duvr|jj?dvksJ|j*dwkr|jj@dxdykr d0ndz|_*n|j*|_*d{|_+|j>|ddrdld||jd4ks+Jd}n|d~vrlt&rkt)|}|j*durH|durH||_*|j*dvsT|j*durk|jd+krk|j+d,krkd.|_+td|n|dvrt&rt)|}|j*dur|dur||_*|j*dvs|j*dur|jd+kr|j+d,krd.|_+td||j>|ddd4d|n|dvrt&rtA|dd}|dur|0dnd}|j*dur|dur||_*|j*d0kr|jd+kr|j+d,krtBddrd.|_+tdnM|dvr |j>|ddrd4d|n>|dkr0|j}tCddtA|dgD}|r/|j>|drd4dn|dvrK|j>|ddrdld||jd4ksKJ|dt jDErYd|_FtdtG}|ogd|vogd|v}|jHs|dvrt1syt&r|js|jId kr|s|jd+krd|_HdSdSdSdSdSdSdSdS)Nr)is_deepseek_nsaMistralLarge3ForCausalLMPixtralForConditionalGenerationr)DeepseekV3ForCausalLMKimiK25ForConditionalGenerationrrGlmMoeDsaForCausalLMrTz_Force NSA prefill to use MLA (i.e. disable MHA_ONE_SHOT) for GlmMoeDsaForCausalLM on Blackwell.r]z0Use nsa attention backend for DeepSeek with DSA.zaContext parallel feature is still under experiment. It has only been verified on Hopper platform.rtrrrzFor in-seq split mode, we have the following restrictions: moe_dense_tp_size == 1, moe_a2a_backend == deepep, ep_size == tp_size, kv_cache_dtype == bf16, batch_size == 1z:For round-robin split mode, dp attention is not supported.rOz}Current multi-machine CP support suffers from precision issues. So context parallel only support Single machine(tp_size == 8)zEEnable Context Parallel opt for deeeseekv3.2-DSA, Setting dp_size == z and moe_dense_tp_size == z , ep_size == z , tp_size == z, kv_cache_dtype == z, moe_a2a_backend  z$DSA with TP mode is active, dp_size=z , tp_size=z, attn_tp_size=z+, attention weights will be sharded across z ranks.z0Setting page size to 1 for DeepSeek DSA on ROCm.rz)Setting page size to 64 for DeepSeek DSA.rUzmCP is only supported for prefill when PD disaggregation, please remove --enable-nsa-prefill-context-parallel.z5Piecewise CUDA graph is enabled, use MLA for prefill.rbzFUse trtllm_mla as attention backend on sm100 for DeepseekV3ForCausalLM)rrFz@Quantization not specified, default to fp8 for DeepSeek on sm100rsr7)rFrMrNrzNUse flashinfer_trtllm as MoE runner backend on sm100 for DeepseekV3ForCausalLMrNEAGLEr~z`Use deep_gemm moe runner and deepep a2a backend for bf16 nextn layer in deepseek fp4 checkpoint.rZzPUse triton fused moe by default for bf16 nextn layer in deepseek fp4 checkpoint.)GptOssForCausalLMrcr_)rZrcr_r`r4z"GptOssForCausalLM requires one of z> attention backend, but got the following backends - Prefill: z - Decode:  zNDisable hybrid SWA memory for GPT-OSS model with trtllm_mha attention backend.rUrziDetected Blackwell and MXFP4 quantization format for GPT-OSS model, enabling FlashInfer MXFP4 MOE kernel.SGLANG_USE_AITERz_Detected ROCm and MXFP4 quantization format for GPT-OSS model, enabling aiter MXFP4 MOE kernel.zTDetected ROCm with SGLANG_USE_AITER for GPT-OSS bf16 model, using triton MOE kernel.rz;Detected GPT-OSS model, enabling triton_kernels MOE kernel.z5Triton kernel MoE is only supported when ep_size == 1MiMoV2FlashForCausalLMzOEnable multi-layer EAGLE speculative decoding for MiMoV2FlashForCausalLM model.z>Spec v2 is enabled for multi-layer EAGLE speculative decoding.rz[Reset swa_full_tokens_ratio to 1.0 for MiMoV2FlashForCausalLM model with hierarchical cachezRDisable hybrid SWA memory for MiMoV2FlashForCausalLM model with hierarchical cacheStep3p5ForCausalLMzKEnable multi-layer EAGLE speculative decoding for Step3p5ForCausalLM model.zWReset swa_full_tokens_ratio to 1.0 for Step3p5ForCausalLM model with hierarchical cachezNDisable hybrid SWA memory for Step3p5ForCausalLM model with hierarchical cacheLlama4r)rcsm100)r_sm90)rehipxpu)rhr)rZzother platformsUse z as attention backend on z for Llama4 model>r_rerZrhrczQfa3, aiter, triton, trtllm_mha or intel_xpu is required for Llama4 model but got >rFrMz?Use flashinfer_trtllm as MoE runner backend on SM100 for Llama4)Gemma2ForCausalLMGemma3ForCausalLMGemma3ForConditionalGenerationGemma3nForCausalLMGemma3nForConditionalGenerationzDisable hybrid SWA memory for  as it is not yet supported.)Exaone4ForCausalLMExaoneMoEForCausalLMz Disabling hybrid SWA memory for )r_rZrcz!One of the attention backends in z is required for , but got )Olmo2ForCausalLMrxr6zMFlashInfer backend can significantly degrade the performance of Olmo3 models.zUsing z as attention backend for r)KimiLinearForCausalLMBailingMoeV2_5ForCausalLMF) model_archsupport_mamba_cache)NemotronHForCausalLM)rLrMrNrelu2rL quant_algoNVFP4rMr)rr support_mamba_cache_extra_buffersm100_default_attention_backendzqNemotronHForCausalLM does not support triton attention backend,as the first layer might not be an attention layer)Qwen3MoeForCausalLM"Qwen3VLMoeForConditionalGeneration)rFrNz9Use flashinfer_trtllm as MoE runner backend on sm100 for )Qwen3NextForCausalLM"Qwen3_5MoeForConditionalGenerationQwen3_5ForConditionalGeneration)Glm4MoeForCausalLMquantization_config quant_methodzflashinfer-pythonz0.6.3zKUse flashinfer_trtllm as MoE runner backend on sm100 for Glm4MoeForCausalLM)FalconH1ForCausalLMJetNemotronForCausalLMJetVLMForConditionalGenerationGraniteMoeHybridForCausalLMcss|]}|dkVqdS)mambaNr3)r layer_typer3r3r ks  z@ServerArgs._handle_model_specific_adjustments.. layer_types)rrr)Lfm2ForCausalLMz^ does not support triton attention backend, as the first layer might not be an attention layerzIOverlap scheduler is disabled when using sparse head for embedding model.H20H200)rrrGlm4MoeLiteForCausalLMrr)Jsglang.srt.configs.model_configrr-rr INSTANCEr hf_config architecturesrrr SGLANG_NSA_FORCE_MLArrYrZis_attention_backend_not_setrUr r$rrrrr}rr|rr@r!rtorchcudaget_device_capabilityrrrrr'rWrVrrr~rarnroSGLANG_NVFP4_CKPT_FP8_NEXTN_MOErbr&get_attention_backendsrrr)r{SGLANG_ENABLE_SPEC_V2rrrsliding_window_patternrr_handle_mamba_radix_cachemlp_hidden_actrgetattrranySGLANG_EMBEDDINGS_SPARSE_HEADrrrrrE)rQrrrrr_rsupported_backendsprefill_attn_backenddecode_attn_backendis_mxfp4_quant_formatplatformaccepted_backendsrr has_mamba device_name is_h20_devicer3r3rr5s*         4                                                                                           z-ServerArgs._handle_model_specific_adjustmentsTrrrrcCstr|jdur|dur||_td|d||s*td|dd|_dS|s8|r8Jd|d|rtsCJd|jdurZ|j |jksZJd |j d |j|j dur|j |j d kssJd |j d |j t t |j t t |j d ksJd t d|j dSdS|js|jdurtdd|_|jdkrtdd|_d|_dSdStd|dd|_dSdS)Nrz# as attention backend on sm100 for zDisabling Radix Cache for rTz(mamba extra_buffer is not supported for z modelzEMamba extra_buffer is only supported on CUDA devices with FLA backendzmamba_track_interval z? must be greater than or equal to speculative_num_draft_tokens rz must be divisible by page_size zxFor SSM models with extra buffer, either FLA_CHUNK_SIZE or page_size must be divisible by the other, got FLA_CHUNK_SIZE=z, self.page_size=zDisabling overlap schedule since mamba no_buffer is not compatible with overlap schedule, try to use --disable-radix-cache if overlap schedule is necessaryrczDisabling radix cache since trtllm_mha does not support page_size = 1, which is required by MambaRadixCache. Try to use --attention-backend triton if radix cache is necessary.Fz5Disabling radix cache since speculative decoding for z' is not supported with radix cache yet.)r'rUrYr rZrenable_mamba_extra_bufferrrgrrrFLA_CHUNK_SIZEminrar)rQrrrrr3r3rrsv            z$ServerArgs._handle_mamba_radix_cachecCs"|jdurtr dnd|_dSdS)Nr6r5)rXr r[r3r3rr6s  z#ServerArgs._handle_sampling_backendcCs|}|}|jdur|j|jkr|j|_|jdur |sOtr*t|r*d|_nOtr?t|r?|jdus;|j dur?d|_n:t rFd|_n3t rKdnd|_n*trVd|_n#tr]d|_nt rv| |j }|dksn|dkrrd|_nd|_nd|_td|jd |jd krtd d |_|jd krtdd |_|jdusJd|jdks|jdkrtdd|_|jdks|jdkrtdd|_|jdks|jdkrtstd|jdvrtd|jdd|_|jdvrtd|jdks|jdks|jdkr#tstd|jdvr#td|jdd|_|jdkr7|jdkr7td d|_|jd!krO|sOtd"|jd#d|_|jdkrb|jd$krb|jd%9_|jd&krz|jd'krztsztd(d |_|jd)kr|jd*krtstd+d|_|jd)kr|jd,vrtd-|jd#d|_t|jd.ddur|jdurd/|_td0n|jd/krtd1|jd2|jd/krtd3d4|_d |_dSdS)5Nr_rcrer6rZrSz%Attention backend not specified. Use z backend by default.r[zFCuda graph is disabled because of using torch native attention backendTr\zDCuda graph is disabled because of using torch Flex Attention backendzKSpeculative decoding is currently not supported with Flex Attention backendrazAFlashMLA only supports a page_size of 64, change page_size to 64.rr^zFCutlass MLA only supports a page_size of 128, change page_size to 128.rbzeTRTLLM MLA backend is only supported on Blackwell GPUs (SM100/SM12x). Please use a different backend.)rrzNTensorRT-LLM MLA only supports page_size of 32 or 64, changing page_size from z to 64.)rfp4_e2m1rr7z[TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, fp4_e2m1, bf16, or auto.z_TRTLLM MHA backend is only supported on Blackwell GPUs (SM100). Please use a different backend.)rSrrzRTensorRT-LLM MHA only supports page_size of 16, 32 or 64, changing page_size from fp8_e5m2zYFlashAttention3 only supports fp8_e4m3 if using FP8; Setting attention backend to triton.r`zaFA4 backend only supports page size 128 for non-MLA model architectures, changing page_size from z to 128.rzg333333?rgrzWThe current platform does not support Intel AMX, will fallback to torch_native backend.rhrzQThe current platform does not support Intel XMX, will fallback to triton backend.)rrrz^Intel XPU attention backend only supports page_size of 32, 64 or 128, changing page_size from dual_chunk_attention_configrdz-Dual chunk attention is turned on by default.zADual chunk attention is enabled, but attention backend is set to z+. Please set it to 'dual_chunk_flash_attn'.zVMixed chunk and radix cache are disabled when using dual-chunk flash attention backendF)rrrWrVrUr"r#r'rarfr!r get_num_kv_headsrrYr rZrrrrXr context_lenrrrr0rrrr)rQrrhead_numr3r3rr7s                                         z2ServerArgs._handle_attention_backend_compatibilitycCs,|jdkrdS|}|\|_|_tr|j|jkr1|jdkr1td|jd|jddS|jdkrd|rNgd}|j|vsLJd|d |jdSgd }|j|vsbJd |d |jdS|r|gd }|j|vszJd |d |jdSgd}|j|vsJd|d |jdSt d)z;Check FP4 KV cache compatibility with the attention backendrNr`z$Attention: Using KV4 with PREFILL = z and DECODE = zF. Compatibility issues are unlikely, but may occur in rare edge cases.)r^r6rbz:KV4 FA4 MLA expects decode_attention_backend to be one of r)rZr[r\z:KV4 FA4 MHA expects decode_attention_backend to be one of )r^r6rbraz/KV4 MLA expects attention_backend to be one of )rZr[r\rcz/KV4 MHA expects attention_backend to be one of z(KV4 is not tested on non-CUDA platforms.) rrrprefill_attention_backend_strdecode_attention_backend_strrrYrZrU RuntimeError)rQrKV4_FA4_MLA_BACKEND_CHOICESKV4_FA4_MHA_BACKEND_CHOICES!KV4_ATTENTION_MLA_BACKEND_CHOICES!KV4_ATTENTION_MHA_BACKEND_CHOICESr3r3rr9sj        z$ServerArgs._handle_kv4_compatibilitycC|jdur d|_dSdS)Nr)rr[r3r3rr:  zServerArgs._handle_page_sizecCstrd|_dSdS)NrS)r!rr[r3r3rr; s z ServerArgs._handle_amd_specificscCr)Nrp)rYr[r3r3rr<rz"ServerArgs._handle_grammar_backendc CsP|jdkr&tr"z ddl}tdWdSttfy!tdwtddS)Nr6rz-Successfully imported FlashInfer mamba modulezLFlashInfer mamba module not available, please check flashinfer installation.)r`r flashinfer.mambarYr  ImportErrorAttributeErrorrX)rQr6r3r3rr8s z ServerArgs._handle_mamba_backendcCs|jdkr)|j|jdksJd|j|j|jdks Jd|jdks)Jd|jdkrb|j|jdks:Jd|j|j|jksGJd|jdksPJd|jdkrd|j|j|jksfJddSdSdS) Nrrz)tp_size must be divisible by attn_cp_sizez3tp_size must be divisible by dp_size * attn_cp_sizez,PP is not supported with context parallelismz(tp_size must be divisible by moe_dp_sizez;ep_size * moe_dp_size must be less than or equal to tp_sizez.ep_size * moe_dp_size must be equal to tp_size)rBrr@rrCr|r[r3r3rr?#s.    z&ServerArgs._handle_context_parallelismcCsz|jdkr d|_d|_|jr/|jd|_|j|jdksJ|j|j|_td|jd|jr9|js;JddSdS)NrFg333333?rzADP attention is enabled. The chunked prefill size is adjusted to z to avoid MoE kernel issues. z;Please enable dp attention when setting enable_dp_lm_head. )r@rrrrrrYrZr[r3r3rr>=s"   z#ServerArgs._handle_data_parallelismcCs|jdkr|jdvrtd|jdd|_|jdkr6|jdvs*Jd|jd |jd |jfvs6Jd |jd krQ|jd vsIJd|jdd|_tdtdrftd|jdvscJdd|_|jdkry|jdvr{|jd ks}JddSdSdS)NrG)r7rzCmxfp8 quantization forces --moe-runner-backend=cutlass. Overriding rrr)rNrMNzInvalid quantization 'z]'. FlashInfer Cutlass MOE supports only: 'modelopt_fp4', 'modelopt_fp8', or bfloat16 (None).rzJThe expert parallel size must be 1 or the same as the tensor parallel sizer)rNrFrMrVNzy'. FlashInfer TRTLLM MOE supports only: 'modelopt_fp4', 'fp8', 'modelopt_fp8', 'compressed-tensors', or bfloat16 (None).TzWFlashInfer TRTLLM MoE is enabled. --disable-shared-experts-fusion is automatically set.SGLANG_CUTLASS_MOEzzSGLANG_CUTLASS_MOE is deprecated, use --moe-runner-backend=cutlass and/or --speculative-moe-runner-backend=cutlass instead)rFrGz9cutlass MoE is only supported with fp8/mxfp8 quantizationz9FP8/MXFP8 Cutlass MoE is only supported with ep_size == 1)rr~rYrZr|rrrr[r3r3rr@OsP          z$ServerArgs._handle_moe_kernel_configcCsd|jdkr |jdkrtdd|_|j|_td|jd|jdkr3|j|_td|jd|jd krF|j|_td |jd|jd kr|j|_td |jdd|_td |jdkrktdtj ddur~t j dtd|jdvsJd|jdkr|j|_d|_tdtd|jd|jtddksJddSdS)Nrnormalz3Cuda graph is disabled because deepep_mode=`normal`TzgDeepEP MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[z].rjziMooncake MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[rzpAscend fused EP MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[r6zoFlashinfer MoE A2A is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[zTFlashinfer MoE A2A is enabled. --disable-shared-experts-fusion is automatically set.r7z/--deepep-mode is ignored for Flashinfer MoE A2ASGLANG_MOE_NVFP4_DISPATCHz?SGLANG_MOE_NVFP4_DISPATCH is set to True for Flashinfer MoE A2A)rzOFlashinfer MoE A2A is only supported with flashinfer_cutlass moe runner backendrmz)auto set deepep_mode=`normal` for MORI EPzeMoRI MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[,SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANKrzkSGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK (default 4096) must be larger or equal to chunked_prefill_size)r}rrYrZrrr|rrmenvironrbr rrr~rrr[r3r3rrAsf                 zServerArgs._handle_a2a_moecCs\|jr|jdurd|_td|js|jdkr |jdur d|_|jr*|jdks,JdSdS)NstatzLEPLB is enabled. The expert_distribution_recorder_mode is automatically set.rstaticr)rrrYrZrrr|r[r3r3rrBs z$ServerArgs._handle_eplb_and_dispatchcCsV|jdur'|jr|jdkrd|_|jdksJd|jdkr)||j|_dSdSdS)Nr7elasticity_awarezMElastic EP requires eplb_algorithm to be set to 'auto' or 'elasticity_aware'.rj)rrr_validate_ib_devicesrr[r3r3rrDs     zServerArgs._handle_elastic_epcCsT|jr |jdur d|_|jdur&|j}dur||_dS|jdur(d|_dSdSdS)Nrr)rrrrrer3r3rrCs     z.ServerArgs._handle_expert_distribution_metricscCs"|jdkrd|_tddSdS)NrTz;Pipeline parallelism is incompatible with overlap schedule.)rrrYrZr[r3r3rrEs z'ServerArgs._handle_pipeline_parallelismcCs|jdkr|jdkrd|_td|js|jrO|jdkrO|jdur%|jn|j}|dkrO|jdurG|s>t r:dnd|_nt rCdnd|_nd|_td|j d krz|jd kr||jdkrbd|_n|jdkrjd |_td |jd |jddSdSdS)Npage_first_directrdirectz;Kernel io backend does not support page first direct layoutr_r6rZzFlashAttention3 decode backend is not compatible with hierarchical cache. Setting hicache_io_backend to vanilla I/O, which may lead to suboptimal performance with small page sizes.rjr page_firstzKMooncake storage backend does not support layer_first layout, switching to z layout for z io backend) rrrYrZrrrVrUrr r'r)rQeffective_decode_backendr3r3rr=sR           zServerArgs._handle_hicachecCs|jdur |jdur d|_ddlm}|jdur$|jdkrdn|j|_n ||jr/Jd|jdkr7d|_|jd vr|jd krI|jrIt d |j durVd |_ t d |jd vrwt jrwd|_t d|jdurv|jdkrvt dnd|_t d|jrd|_t d|jjd}|dvr|jdur|j|_|j|_n |dvrt d|jdur|jdur|jdusJt|\|_|_|_|jdks|jdks|jdkr|jdkrt d|jdkr|j|jdkrt d|jd|_|jdkr|jdkr|jdvrt d|jdkrm|j dst d|j dur-d |_ t d d|_d|_|j!|_|jdurA|j"|_t d |jdkre|jdkre|jd!kret d"|jd#|jd$|jrot d%dSdS)&Nmainr)MoeRunnerBackendrr7zcCurrently speculative MoE runner backend cannot be flashinfer_trtllm for risk in some draft models.NEXTNr)rEAGLE3r}r}zHCurrently standalone speculative decoding does not support dp attention.rzMax running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests.Fz[Spec v2 is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on.rzBSpec v2 currently only supports topk = 1 for speculative decoding.TzOverlap scheduler is disabled when spec v2 is off or using unsupported speculative algorithm. You can set env SGLANG_ENABLE_SPEC_V2=True to enable the experimental overlap scheduler. zNMixed chunked prefill is disabled because of using eagle speculative decoding.) DeepseekV32ForCausalLMrrrrBailingMoeForCausalLMBailingMoeV2ForCausalLMrrrrzCDeepSeek MTP does not require setting speculative_draft_model_path.rczCtrtllm_mha backend only supports topk = 1 for speculative decoding.zfspeculative_num_draft_tokens is adjusted to speculative_num_steps + 1 when speculative_eagle_topk == 1)r6r_zspeculative_eagle_topk > 1 with page_size > 1 is unstable and produces incorrect results for paged attention backends. This combination is only supported for the 'flashinfer' backend.r~rz5Ngram speculative decoding only supports CUDA device.ziThe overlap scheduler and mixed chunked prefill are disabled because of using ngram speculative decoding.r6zspeculative_eagle_topk(z) > 1 with page_size(z) > 1 is unstable and produces incorrect results for paged attention backends. This combination is only supported for the 'flashinfer' backend.zCCurrently ngram speculative decoding does not support dp attention.)#rbrcsglang.srt.layers.moe.utilsr!rnr~is_flashinfer_trtllmrarrXrrYrZr rrbrrfrrrrrrrergauto_choose_speculative_paramsrUrVrWrr startswithrurs)rQr!rr3r3rrF s                               #z'ServerArgs._handle_speculative_decodingcCs|jdks |jdkrt|jrd|_|_t|jrd|_|jdur%g|_|jdkra|jdus4|jdur=t dd|_n$|j durP|j dkrPt dd|_n| sa|j dkrat d d|_|j rk| |_ dSdS) Nr7r=rArBzXFallback load_format to 'auto' due to incomplete remote instance weight loader settings.rziFallback load_format to 'auto' due to incomplete remote instance weight loader NCCL group ports settings.transfer_enginezQFallback load_format to 'auto' due to 'transfer_engine' backend is not supported.)rr1rrr%rrrrYrZrrvalidate_transfer_enginerr[r3r3rrG s@         zServerArgs._handle_load_formatcCs|jdkr!|jdusJd|jdusJdd|_tddS|jdkrQ|jdur/|j|_|jdur8|j|_|j|_ | |j|j|j sSd|_ tddSdSdS)NrUz--enable-prefix-mm-cache requires --encoder-only to be enabledz6Cannot set --encoder-only and --language-only togetherrzKCannot set --encoder-only and --disaggregation-mode prefill/decode togetherrz?requires at least one encoder urls to be set via --encoder-urlsrj)rkrU) r%rrXrrrrr rrrr[r3r3rrI s*      z)ServerArgs._handle_encoder_disaggregation device_strcs|dur tddSdd|dD}t|dkrtdt|tt|kr0td|d }tj|s@t d |d tt |tdkrTt d |fd d|D}t|dkrotd|dt d |S)a Validate IB devices before passing to mooncake. Args: device_str: Comma-separated IB device names (e.g., "mlx5_0,mlx5_1") Returns: Normalized comma-separated string of validated device names, or None if input is None. NzMNo IB devices specified for Mooncake backend, falling back to auto discovery.cSsg|] }|r|qSr3)striprdr3r3rr8 sz3ServerArgs._validate_ib_devices..,rzNo valid IB devices specifiedz Duplicate IB devices specified: z/sys/class/infinibandz!InfiniBand sysfs path not found: z1. Please ensure InfiniBand drivers are installed.zNo IB devices found in csg|]}|vr|qSr3r3r0available_devicesr3rrM zInvalid IB devices specified: z. Available devices: ) rYrZsplitrrXrrmrnisdirr listdirsortedjoin)rQr.devices ib_sysfs_pathinvalid_devicesr3r3rr' s4       zServerArgs._validate_ib_devicescCsv|jr |jr td|jr7|jdkrtd|jdd|_|jr*tdd|_|jr9tdd|_dSdSdS)NzCannot enable both --enable-tokenizer-batch-encode and --enable-dynamic-batch-tokenizer. Please choose one tokenizer batching approach.rz_skip_tokenizer_init=True disables tokenizer workers; forcing tokenizer_worker_num=1 (requested z).zOskip_tokenizer_init=True ignores --enable-tokenizer-batch-encode; disabling it.FzPskip_tokenizer_init=True ignores --enable-dynamic-batch-tokenizer; disabling it.)rrrXrrrYrZr[r3r3rrJV s0   z%ServerArgs._handle_tokenizer_batchingcCsdtj|jrdnd|jdurtj|jtj|jrdndtj|j r-ddSddS)N10) r SGLANG_ENABLE_TORCH_COMPILErrrSGLANG_MAMBA_SSM_DTYPE"SGLANG_DISABLE_OUTLINES_DISK_CACHEr%SGLANG_ENABLE_DETERMINISTIC_INFERENCErr[r3r3rrKq s  z(ServerArgs._handle_environment_variablescCsr|jr |jr td|jr$|jdkrtd|jdkr$tjr$tdd|jkr2dks7tdtddS)NzThe arguments enable-hierarchical-cache and disable-radix-cache are mutually exclusive and cannot be used at the same time. Please use only one of them.rUz\The argument disaggregation-decode-enable-offload-kvcache is only supported for decode side.zTSpec v2 and decode offload kv cache are incompatible and cannot be enabled together.rrz4--swa-full-tokens-ratio should be in range (0, 1.0].) rrrXrrr rrbrr[r3r3rrL| s(   z&ServerArgs._handle_cache_compatibilitycCs|jdurtdd|_dtjd<dtjd<|jrd|_tdd }t|jt j krFz| j }|j d }|d v}Wn tyEYnw|jdurltsQtr[|rWd |_nd |_nd|_td|jdtdn|jtvr}tdtd|jd|r|jdvrtdtd|jd|jtvrd|_td|jd|jdkrtrtddSdtjd<d|_tddSdSdS)Nz>Enable deterministic inference because of rl_on_policy_target.Tr?SGLANG_VLM_CACHE_SIZE_MBr>rCr5z?Sampling backend is set to pytorch for deterministic inference.Fr)DeepseekV2ForCausalLMrr$rrrrZr6r_z2Attention backend not specified. Falling back to 'zT' for deterministic inference. You can explicitly set --attention-backend to one of rzCurrently only z] attention backends are supported for deterministic inference, but you explicitly specified 'z'.)r_rZze attention backends are supported for deterministic inference with DeepSeek models. But you're using z-Currently radix cache is not compatible with zS attention backend for deterministic inference. It will be supported in the future.rz9AMD/ROCm: Using 1-stage all-reduce kernel (deterministic)zallreduce:tree NCCL_ALGOztNCCL_ALGO is set to 'allreduce:tree' and custom all reduce is disabled for deterministic inference when TP size > 1.)rrYrZrrmrrXr-rr rrrr ExceptionrUr'r(rrXrrrr!r r)rQis_deepseek_modelrrr3r3rrM s|               @z*ServerArgs._handle_deterministic_inferencecCs|jdurdStr#|jstdd|_|jdvr"tdd|_n|js3|jdkr3tdd|_|js>tdd|_|jsItd d|_|jd ksVtd d |_|j ratd d |_ |j dkrntdd|_ |j r{tdd |_ dSdS)Nz>Cuda graph is disabled for diffusion LLM inference on AMD GPUsT)rZrezJAttention backend is set to triton for diffusion LLM inference on AMD GPUsrZr6z`Attention backend is set to flashinfer because of enabling cuda graph in diffusion LLM inferencezEOverlap schedule is disabled because of using diffusion LLM inferencez@Radix cache is disabled because of using diffusion LLM inferencerzIPipeline parallelism is disabled because of using diffusion LLM inferencez;Currently LoRA is not supported by diffusion LLM inference.FrzECurrently disaggregation is not supported by diffusion LLM inference.zKMixed chunked prefill is disabled because of using diffusion LLM inference.) rr!rrYrZrUrrrrIrrr[r3r3rrN sf      z!ServerArgs._handle_dllm_inferencecCs|jdurtdd|_d|_|jrBt|jtr t |j|_t|jt rBhd}|j D]}||vrAt d|dt |q/|jrTt|jtrVt |j|_dSdSdS)NzKCuda graph and server warmup are disabled because of using tensor dump modeT>audioimagevideozInvalid modality 'z9' in --limit-mm-data-per-request.Allowed modalities are: )rrYrZrrr( isinstancerjsonloadsdictkeysrXrrH)rQallowed_modalitiesmodalityr3r3rrP$ s8     z$ServerArgs._handle_other_validationscCs,tr|jdurtdd|_dSdSdS)Nz%Set soft_watchdog_timeout since in CIr)r2rrYr r[r3r3rrOD s  zServerArgs._handle_debug_utilsparserargparse.ArgumentParserc Cs|jddtddd|jdttjdd|jd ttjd d gd d |jdttjdd|jdddd|jdttjtdd |jdtdtj d|jdddd|jdttj dd|jdddd|jdtj dd d!|jd"tdd#d|jd$ttj d%d|jd&ttj d'd|jd(ttjd)d|jd*ttjd+d|jd,dd-d|jd.dd/d|jd0td1d2d3|jd4ttjd5d|jd6dd7d|jd8ttjgd9d:d |jd;ttjtdd|jd?ttjgd@dAd |jdBddCd|jdDttjdEd|jdFttjdGd|jdHttjdId|jdJttjdKd|jdLdtjdMdN|jdOttjdPd|jdQttjdRd|jdSttjdTd|jdUttjdVd|jdWttj dXd|jdYttj!dZd|jd[ttj"d\d|jd]dtj#d^dN|jd_ttj$d`d|jdattj%gdbdcd |jdddtj&dedN|jdfdtj'dgdN|jdhdtj(didN|jdjttj)dkd|jdlttj*dmd|jdnttj+dod|jdpt,dqd|jdrttj-dsd|jdtddud|jdvtt.tj/dwdx|jdyddzd|jd{ttj0d|d|jd}tdd~d|jdtdddd|jdtdddd|jdttj1dd|jddttj2dd|jddttj3dd|jddttj4dd|jddttj5dd|jdttj6dd|jdttj7dd|jdttj8dd|jdddd|jdttj9dd|jdttj:dd|jdddd|jdttj;dd|jdttjdd|jdtddddd|jdttj?dd|jdttj@dd|jdddd|jddd|jdttjAdd|jdttjBdd|jdddd|jdttjCdgdd|jdttjDddgdd |jdtdtjEdd|jdtdtFtGdd|jdttjHdd|jdddd|jdddd|jdddd|jdttjIdd|jdtdtjJdd|jdtKjLtjMdd|jdtdtjNdd|jdtdtjOdd|jdtdtjPdd|jddtjQddNd}|jdtdtjRd|d|jdtdtjSd|d|jdttjTdd|jdttjUdd|jddtjVddN|jdtddd|jdddd|jdtddd|jdddd|jdttjWdd|jdttjXdd|jdttjYdd|jdttjZdd|jdttj[dd|jdttj\dd|jdttj]dd|jdttj^dd|jdttj_dd|jdddd|jdttFt`jabtjcd tFt`jabd dxtFtdjeb}|jd t|tjfd |d dx|jd tddd|jdtddgtjgddx|jddttjhdd|jdttjidgdd|jdt,dd|jddtdd|jdttjjd d|jd!ttjkd"d|jd#td$tjld|jd%tKjLd&d|jd'tjmdd(d!|jd)tjndd*d!|jd+tjotd,d-|jd.ttptqgddd/d0|jd1tddtrd2d3|jd4td5d6d|jd7ttjsd8d|jd9ttjtd:d;gddx|jd?ttjwgd@dAd |jdBttxtjydCdx|jdDttxtjzdEdx|jdFttxtj{dGdx|jdHtt|tj}dIdx|jdJtt~tjdKdx|jdLtgdMtjdNdx|jdOtjttdPdQ|jdRtjttdSdQ|jdTtttjdUdVdW|jdXtttjdYdZdW|jd[tjdd\d!|jd]tgd^d_d`|jdadbtdcd|jddtdded|jdfttjtdgd |jdhtditjd|jdjtdktjd|jdltdmtjd|jdntdotjd|jdptdqtjd|jdrtdstjd|jdttdudvgdwtjdx|jdytdztjd|jd{tttjd|dx|jd}tttjd~dx|jdtttjddx|jdttjdd|jdttjdd|jdttjdd|jdttjdd|jdtddgtjddx|jdttjdd|jdttjdd|jdddd|jdddttjdd|jdtttjddx|jdtttjddx|jdtddgtjddx|jdddd|jdtgdd ddx|jdttjdd|jdttjdd|jdttjdd|jdddd|jdttjdd|jdttjdd|jdttjdd|jdttjdd|jdttjdd|jdttjdd|jdddd|jdttjdd|jdttjdd|jdttjddgdd |jdttjdd|jdttjdd|jdtdtdd |jdttjdd|jdtttjddx|jdttjdd|jdtttjddx|jdddd|jdttjdd|jdttjdd|jdtgdעtjddx|jdtgdڢtjddx|jdtgdݢtjddx|jdddd|jdtgdtjddx|jdtgdtjddx|jdttjdd|jdttjdd|jdddd|jdtdd|jdtddd|jdtdd|jdtddd|jdtdd|jdttjdd|jdttjdd|jdttjÐdd|jdddd|jdttjĐdd|jdttjŐdd|jdttjƐdd|jdttjǐdd|jd ttjȐd d|jd ttjɐd d|jd ttjʐdd|jdttjːdd|jdttj̐dd|jdttj͐dd|jdttjΐdd|jdddd|jdttjϐdd|jdtddd|jdddd|jd dd!d|jd"dd#d|jd$dd%d|jd&dd'd|jd(dd)d|jd*dd+d|jd,dd-d|jd.dd/d|jd0dd1d|jd2dd3d|jd4dd5d|jd6dd7d|jd8dd9d|jd:dd;d|jddd?d|jd@ddAd|jdBddCd|jdDddEd|jdFttjАdGd|jdHddId|jdJddKd|jdLddMd|jdNtddOd|jdPttjѐdQdRdSgd|jdTttjҐdUd|jdVttjӐdWd|jdXttjԐdYd|jdZdd[d|jd\dd]d|jd^dd_d|jd`ttjՐdad|jdbttj֐dcd|jddttjאded|jdfddgd|jdhddid|jdjddkd|jdlddmd|jdnddod|jdpddqd|jdrddsd|jdtddud|jdvddwd|jdxddyd|jdzdd{d|jd|dd}d|jd~ddd|jdttjؐdd|jdtddd|jdddd|jdttjtڐdd |jdddd|jdddd|jdttjtܐdd |jdddd|jdddd|jdddd|jdttjݐdd|jdttjސdd|jdttjߐdd|jdtddd|jdttjdd|jdttjdd|jdttjgddd |jdttjtdd |jdttjdd|jdttjdd|jdttjdd|jdttjdd|jdttjdd|jdddd|jdttjdd|jdttjdd|jdddd|jdddd|jdttjtdd |jddtgdd|jdtdddd|jdddd|jdttjdd|jdttjdd|jdttjdd|jdtdɐdgtjddx|jdddd|jdddd|jdtddd|jdttjdd|jdtdՐd|jdttjdd|jdttjdd|jddtjddN|jdtKjLtjdd|jddtjddN|jdtKjLtjdd|jdttjdd|jdttjdd|jddtjddN|jdttjdddS(Nz --model-pathz--modelzTThe path of the model weights. This can be a local folder or a Hugging Face repo ID.T)typehelprequiredz--tokenizer-pathzThe path of the tokenizer.)rUr3rVz--tokenizer-moder7slowzoTokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer.)rUr3rrVz--tokenizer-worker-numz(The worker num of the tokenizer manager.z--skip-tokenizer-init store_truezCIf set, skip init tokenizer and pass input_ids in generate request.)actionrVz --load-formataThe format of the model weights to load. "auto" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available. "pt" will load the weights in the pytorch bin format. "safetensors" will load the weights in the safetensors format. "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading. "dummy" will initialize the weights with random values, which is mainly for profiling."gguf" will load the weights in the gguf format. "bitsandbytes" will load the weights using bitsandbytes quantization."layered" loads weights layer by layer so that one can quantize a layer before loading another to make the peak memory envelope smaller.z--model-loader-extra-configzoExtra config for model loader. This will be passed to the model loader corresponding to the chosen load_format.)rUrVr3z--trust-remote-codezYWhether or not to allow for custom models defined on the Hub in their own modeling files.z--context-lengthzoThe model's maximum context length. Defaults to None (will use the value from the model's config.json instead).z--is-embeddingz0Whether to use a CausalLM as an embedding model.z--enable-multimodalzzEnable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen)r3rZrVz --revisionzThe specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.z --model-implagWhich implementation of the model to use. * "auto" will try to use the SGLang implementation if it exists and fall back to the Transformers implementation if no SGLang implementation is available. * "sglang" will use the SGLang model implementation. * "transformers" will use the Transformers model * "mindspore" will use the MindSpore model implementation. z--hostzThe host of the HTTP server.z--portzThe port of the HTTP server.z--fastapi-root-pathz)App is behind a path based routing proxy.z --grpc-modez/If set, use gRPC server instead of HTTP server.z--skip-server-warmupzIf set, skip warmup.z --warmupsFzSpecify custom warmup functions (csv) to run before server starts eg. --warmups=warmup_name1,warmup_name2 will run the functions `warmup_name1` and `warmup_name2` specified in warmup.py before the server starts listening for requests)rUrWrVz --nccl-portzKThe port for NCCL distributed environment setup. Defaults to a random port.z---checkpoint-engine-wait-weights-before-readyzIf set, the server will wait for initial weights to be loaded via checkpoint-engine or other update methods before serving inference requests.z--dtype)r7halfrrrragData type for model weights and activations. * "auto" will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models. * "half" for FP16. Recommended for AWQ quantization. * "float16" is the same as "half". * "bfloat16" for a balance between precision and range. * "float" is shorthand for FP32 precision. * "float32" for FP32 precision.z--quantizationzThe quantization method.z--quantization-param-pathzPath to the JSON file containing the KV cache scaling factors. This should generally be supplied, when KV cache dtype is FP8. Otherwise, KV cache scaling factors default to 1.0, which may cause accuracy issues. z--kv-cache-dtype)r7rrrrrzData type for kv cache storage. "auto" will use model data type. "bf16" or "bfloat16" for BF16 KV cache. "fp8_e5m2" and "fp8_e4m3" are supported for CUDA 11.8+. "fp4_e2m1" (only mxfp4) is supported for CUDA 12.8+ and PyTorch 2.8.0+z--enable-fp32-lm-headz1If set, the LM head outputs (logits) are in FP32.z--modelopt-quantzThe ModelOpt quantization configuration. Supported values: 'fp8', 'int4_awq', 'w4a8_awq', 'nvfp4', 'nvfp4_awq'. This requires the NVIDIA Model Optimizer library to be installed: pip install nvidia-modeloptz"--modelopt-checkpoint-restore-pathzPath to restore a previously saved ModelOpt quantized checkpoint. If provided, the quantization process will be skipped and the model will be loaded from this checkpoint.z--modelopt-checkpoint-save-pathzzPath to save the ModelOpt quantized checkpoint after quantization. This allows reusing the quantized model in future runs.z--modelopt-export-pathzPath to export the quantized model in HuggingFace format after ModelOpt quantization. The exported model can then be used directly with SGLang for inference. If not provided, the model will not be exported.z--quantize-and-servezQuantize the model with ModelOpt and immediately serve it without exporting. This is useful for development and prototyping. For production, it's recommended to use separate quantization and deployment steps.)rZr3rVz--rl-quant-profilezUPath to the FlashRL quantization profile. Required when using --load-format flash_rl.z--mem-fraction-staticzThe fraction of the memory used for static allocation (model weights and KV cache memory pool). Use a smaller value if you see out-of-memory errors.z--max-running-requestsz'The maximum number of running requests.z--max-queued-requestsz]The maximum number of queued requests. This option is ignored when using disaggregation-mode.z--max-total-tokenszThe maximum number of tokens in the memory pool. If not specified, it will be automatically calculated based on the memory usage fraction. This option is typically used for development and debugging purposes.z--chunked-prefill-sizeztThe maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked prefill.z--prefill-max-requestszWThe maximum number of requests in a prefill batch. If not specified, there is no limit.z--enable-dynamic-chunkingzEnable dynamic chunk size adjustment for pipeline parallelism. When enabled, chunk sizes are dynamically calculated based on fitted function to maintain consistent execution time across chunks.z--max-prefill-tokenszThe maximum number of tokens in a prefill batch. The real bound will be the maximum of this value and the model's maximum context length.z--schedule-policy)lpmrkrz dfs-weightlofpriorityz routing-keyz&The scheduling policy of the requests.z--enable-priority-schedulingzlEnable priority scheduling. Requests with higher priority integer values will be scheduled first by default.z!--abort-on-priority-when-disabledzTIf set, abort requests that specify a priority when priority scheduling is disabled.z$--schedule-low-priority-values-firstz~If specified with --enable-priority-scheduling, the scheduler will schedule requests with lower priority integer values first.z*--priority-scheduling-preemption-thresholdz_Minimum difference in priorities for an incoming request to have to preempt running request(s).z--schedule-conservativenesszHow conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see requests being retracted frequently.z --page-sizezThe number of tokens in a page.z--hybrid-kvcache-ratioz[Note: --hybrid-kvcache-ratio is deprecated now. Please use --swa-full-tokens-ratio instead.z--swa-full-tokens-ratiozThe ratio of SWA layer KV tokens / full layer KV tokens, regardless of the number of swa:full layers. It should be between 0 and 1. E.g. 0.5 means if each swa layer has 50 tokens, then each full layer has 100 tokens.z--disable-hybrid-swa-memoryz#Disable the hybrid SWA memory pool.z--radix-eviction-policyzqThe eviction policy of radix trees. 'lru' stands for Least Recently Used, 'lfu' stands for Least Frequently Used.)rUrr3rVz--enable-prefill-delayerz ' generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets [984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'custom ...' uses custom bucket values (e.g., 'custom 10 50 100 500').--prompt-tokens-bucketsz#The buckets rule of prompt tokens. --generation-tokens-bucketsz2The buckets rule for generation tokens histogram. z--gc-warning-threshold-secszqThe threshold for long GC warning. If a GC takes longer than this, a warning will be logged. Set to 0 to disable.z--decode-log-intervalz!The log interval of decode batch.z#--enable-request-time-stats-loggingz%Enable per request time stats loggingz--kv-events-configzmConfig in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used.z--enable-tracezEnable opentelemetry tracez--otlp-traces-endpointr,zUConfig opentelemetry collector endpoint if --enable-trace is set. format: :z--export-metrics-to-filezdExport performance metrics for each request to local file (e.g. for forwarding to external systems).z--export-metrics-to-file-dirziDirectory path for writing performance metrics files (required when --export-metrics-to-file is enabled).z --api-keyzOSet API key of the server. It is also used in the OpenAI API compatible server.z--admin-api-keyzSet admin API key for sensitive management endpoints (e.g. /clear_hicache_storage_backend). When set, admin endpoints require this key and do NOT accept --api-key.z--served-model-namezPOverride the model name returned by the v1/models endpoint in OpenAI API server.z--weight-versionzQVersion identifier for the model weights. Defaults to 'default' if not specified.z--chat-templatezzThe buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.z--hf-chat-template-namezWhen the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), specify which named template to use. If not set, the first available template is used.z--completion-templatezThe buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently.z--file-storage-pathz(The path of the file storage in backend.z--enable-cache-reportzVReturn number of cached tokens in usage.prompt_tokens_details for each openai request.z--reasoning-parserz@Specify the parser for reasoning models, supported parsers are: rz--tool-call-parserzISpecify the parser for handling tool-call interactions. Options include: z --tool-serverzEither 'demo' or a comma-separated list of tool server urls to use for the model. If not specified, no tool server will be used.z--sampling-defaultsopenair>zWhere to get default sampling parameters. 'openai' uses SGLang/OpenAI defaults (temperature=1.0, top_p=1.0, etc.). 'model' uses the model's generation_config.json to get the recommended sampling parameters if available. Default is 'model'.z--data-parallel-sizez --dp-sizezThe data parallelism size.z--load-balance-methodz1The load balancing strategy for data parallelism.)r7rWrVtotal_requests total_tokensz--prefill-round-robin-balancez6Note: --prefill-round-robin-balance is deprecated now.z--dist-init-addrz--nccl-init-addrzRThe host address for initializing distributed backend (e.g., `192.168.0.2:25000`).)rUrVz--nnodeszThe number of nodes.z --node-rankzThe node rank.z--json-model-override-argszQA dictionary in JSON string format used to override default model configurations.z--preferred-sampling-paramszIjson-formatted sampling settings that will be returned in /get_model_infoz --enable-lorazEnable LoRA support for the model. This argument is automatically set to True if `--lora-paths` is provided for backward compatibility.z--enable-lora-overlap-loadingzEnable asynchronous LoRA weight loading in order to overlap H2D transfers with GPU compute. This should be enabled if you find that your LoRA workloads are bottlenecked by adapter weight loading, for example when frequently loading large LoRA adapters.z--max-lora-rankzThe maximum rank of LoRA adapters. If not specified, it will be automatically inferred from the adapters provided in --lora-paths.)r3rUrVz--lora-target-moduleszThe union set of all target modules where LoRA should be applied. If not specified, it will be automatically inferred from the adapters provided in --lora-paths. If 'all' is specified, all supported modules will be targeted.)rUrr`r3rVz --lora-pathszThe list of LoRA adapters to load. Each adapter must be specified in one of the following formats: | = | JSON with schema {"lora_name":str,"lora_path":str,"pinned":bool})rUr`r3rZrVz--max-loras-per-batchrOzJMaximum number of adapters for a running batch, include base-only request.z--max-loaded-loraszIf specified, it limits the maximum number of LoRA adapters loaded in CPU memory at a time. The value must be greater than or equal to `--max-loras-per-batch`.z--lora-eviction-policyrvfifozLoRA adapter eviction policy when memory pool is full. 'lru': Least Recently Used (default, better cache efficiency). 'fifo': First-In-First-Out.z--lora-backendz1Choose the kernel backend for multi-LoRA serving.z--max-lora-chunk-size)rSrrrzMaximum chunk size for the ChunkedSGMV LoRA backend. Only used when --lora-backend is 'csgmv'. Choosing a larger value might improve performance.z--attention-backendz(Choose the kernels for attention layers.z--prefill-attention-backendzYChoose the kernels for prefill attention layers (have priority over --attention-backend).z--decode-attention-backendzXChoose the kernels for decode attention layers (have priority over --attention-backend).z--sampling-backendz'Choose the kernels for sampling layers.z--grammar-backendz/Choose the backend for grammar-guided decoding.z--mm-attention-backend)sdpar_r` triton_attn ascend_attn aiter_attnz!Set multimodal attention backend.z--nsa-prefill-backendzYNSA prefill backend. If not specified, auto-detects based on hardware and kv_cache_dtype.)r3rUrrVz--nsa-decode-backendzXNSA decode backend. If not specified, auto-detects based on hardware and kv_cache_dtype.z--fp8-gemm-backendr[aChoose the runner backend for Blockwise FP8 GEMM operations. Options: 'auto' (default, auto-selects based on hardware), 'deep_gemm' (JIT-compiled; enabled by default on NVIDIA Hopper (SM90) and Blackwell (SM100) when DeepGEMM is installed), 'flashinfer_trtllm' (optimal for Blackwell and low-latency), 'flashinfer_deepgemm' (Hopper SM90 only; uses swapAB optimization for small M dimensions in decoding), 'cutlass' (optimal for Hopper/Blackwell GPUs and high-throughput), 'triton' (fallback, widely compatible), 'aiter' (ROCm only). NOTE: This replaces the deprecated environment variables SGLANG_ENABLE_FLASHINFER_FP8_GEMM and SGLANG_SUPPORT_CUTLASS_BLOCK_FP8.)rUrr3destrVz--fp4-gemm-backendr\aChoose the runner backend for NVFP4 GEMM operations. Options: 'flashinfer_cutlass' (default), 'auto' (auto-selects between flashinfer_cudnn/flashinfer_cutlass based on CUDA/cuDNN version), 'flashinfer_cudnn' (FlashInfer cuDNN backend, optimal on CUDA 13+ with cuDNN 9.15+), 'flashinfer_trtllm' (FlashInfer TensorRT-LLM backend, requires different weight preparation with shuffling). NOTE: This replaces the deprecated environment variable SGLANG_FLASHINFER_FP4_GEMM_BACKEND.z--disable-flashinfer-autotunezDisable FlashInfer autotuning.z--speculative-algorithm)rr#r"r}r~zSpeculative algorithm.)rUrrVz--speculative-draft-model-pathz--speculative-draft-modelzZThe path of the draft model weights. This can be a local folder or a Hugging Face repo ID.z"--speculative-draft-model-revisionzThe specific draft model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.z--speculative-draft-load-formatzThe format of the draft model weights to load. If not specified, will use the same format as --load-format. Use 'dummy' to initialize draft model weights with random values for profiling.z--speculative-num-stepszEThe number of steps sampled from draft model in Speculative Decoding.z--speculative-eagle-topkzFThe number of tokens sampled from the draft model in eagle2 each step.z--speculative-num-draft-tokenszJThe number of tokens sampled from the draft model in Speculative Decoding.z%--speculative-accept-threshold-singlez[Accept a draft token if its probability in the target model is greater than this threshold.z"--speculative-accept-threshold-acczmThe accept probability of a draft token is raised from its target probability p to min(1, p / threshold_acc).z--speculative-token-mapz0The path of the draft model's small vocab table.z--speculative-attention-moderkrUzAttention backend for speculative decoding operations (both target verify and draft extend). Can be one of 'prefill' (default) or 'decode'.)rUrrVr3z%--speculative-draft-attention-backendz4Attention backend for speculative decoding drafting.z --speculative-moe-runner-backendz:Choose the runner backend for MoE in speculative decoding.z--speculative-moe-a2a-backendz6Choose the backend for MoE A2A in speculative decodingz&--speculative-draft-model-quantizationz.The quantization method for speculative model.z)--speculative-ngram-min-match-window-sizezKThe minimum window size for pattern matching in ngram speculative decoding.z)--speculative-ngram-max-match-window-sizezKThe maximum window size for pattern matching in ngram speculative decoding.z#--speculative-ngram-min-bfs-breadthzQThe minimum breadth for BFS (Breadth-First Search) in ngram speculative decoding.z#--speculative-ngram-max-bfs-breadthzQThe maximum breadth for BFS (Breadth-First Search) in ngram speculative decoding.z--speculative-ngram-match-typervPROBzThe match type for cache tree.z!--speculative-ngram-branch-lengthz1The branch length for ngram speculative decoding.z--speculative-ngram-capacityz2The cache capacity for ngram speculative decoding.z--enable-multi-layer-eaglez.Enable multi-layer Eagle speculative decoding.z--expert-parallel-sizez --ep-sizez--epzThe expert parallelism size.z--moe-a2a-backendzChoose the backend for MoE A2A.z--moe-runner-backendz"Choose the runner backend for MoE.z --flashinfer-mxfp4-moe-precisionr3rz8Choose the computation precision of flashinfer mxfp4 moez$--enable-flashinfer-allreduce-fusionz9Enable FlashInfer allreduce fusion with Residual RMSNorm.z --deepep-mode)r low_latencyr7zSelect the mode when enable DeepEP MoE, could be `normal`, `low_latency` or `auto`. Default is `auto`, which means `low_latency` for decode batch and `normal` for prefill batch.z--ep-num-redundant-expertsz=Allocate this number of redundant experts in expert parallel.z--ep-dispatch-algorithmzGThe algorithm to choose ranks for redundant experts in expert parallel.z--init-expert-locationzInitial location of EP experts.z --enable-eplbzEnable EPLB algorithmz--eplb-algorithmzChosen EPLB algorithmz--eplb-rebalance-num-iterationsz@Number of iterations to automatically trigger a EPLB re-balance.z!--eplb-rebalance-layers-per-chunkz/Number of layers to rebalance per forward pass.z,--eplb-min-rebalancing-utilization-thresholdzkMinimum threshold for GPU average utilization to trigger EPLB rebalancing. Must be in the range [0.0, 1.0].z#--expert-distribution-recorder-modez%Mode of expert distribution recorder.z*--expert-distribution-recorder-buffer-sizezZCircular buffer size of expert distribution recorder. Set to -1 to denote infinite buffer.z$--enable-expert-distribution-metricsz.Enable logging metrics for expert balancednessz--deepep-configznTuned DeepEP config suitable for your own cluster. It can be either a string with JSON content or a file path.z--moe-dense-tp-sizezTP size for MoE dense MLP layers. This flag is useful when, with large TP size, there are errors caused by weights in MLP layers having dimension smaller than the min dimension GEMM supports.z--elastic-ep-backendrsrjz[Specify the collective communication backend for elastic EP. Currently supports 'mooncake'.z--mooncake-ib-devicezThe InfiniBand devices for Mooncake Backend transfer, accepts multiple comma-separated devices (e.g., --mooncake-ib-device mlx5_0,mlx5_1). Default is None, which triggers automatic device detection when Mooncake Backend is enabled.z--max-mamba-cache-sizez$The maximum size of the mamba cache.z--mamba-ssm-dtypezmThe data type of the SSM states in mamba cache. If not set, will be read from model config (mamba_ssm_dtype).z--mamba-full-memory-ratioz8The ratio of mamba state memory to full kv cache memory.z--mamba-scheduler-strategyz*The strategy to use for mamba radix cache.z--mamba-track-intervalz4The interval to track the mamba state during decode.z--mamba-backendzChoose the kernel backend for Mamba SSM operations. Default is 'triton'. Options: 'triton' (default), 'flashinfer' (requires FlashInfer with Mamba support).z--enable-hierarchical-cachezEnable hierarchical cachez--hicache-ratiozNThe ratio of the size of host KV cache memory pool to the size of device pool.z--hicache-sizezaThe size of host KV cache memory pool in gigabytes, which will override the hicache_ratio if set.z--hicache-write-policy) write_backrwrite_through_selectivez'The write policy of hierarchical cache.z--hicache-io-backend)rr kernel_ascendz8The IO backend for KV cache transfer between CPU and GPUz--hicache-mem-layout)rrrpage_first_kv_split page_headz6The layout of host memory pool for hierarchical cache.z--disable-hicache-numa-detectzrDisable binding the process to the NUMA node closest to the active CUDA device when hierarchical cache is enabled.z--hicache-storage-backend)filerjhf3fsrkaibrixdynamiceicaThe storage backend for hierarchical KV cache. Built-in backends: file, mooncake, hf3fs, nixl, aibrix. For dynamic backend, use --hicache-storage-backend-extra-config to specify: backend_name (custom name), module_path (Python module path), class_name (backend class name).z!--hicache-storage-prefetch-policy)r wait_completetimeoutz>Control when prefetching from the storage backend should stop.z&--hicache-storage-backend-extra-configzA dictionary in JSON string format, or a string starting with a leading '@' and a config file in JSON/YAML/TOML format, containing extra configuration for the storage backend.z,--hierarchical-sparse-attention-extra-configaJA dictionary in JSON string format for hierarchical sparse attention configuration. Required fields: algorithm (str), backend (str). All other fields are algorithm-specific and passed to the algorithm constructor. Example: '{"algorithm": "quest", "backend": "flashattention", "sparsity_ratio": 0.7, "min_sparse_prompt_len": 2048}'z--enable-lmcachez;Using LMCache as an alternative hierarchical cache solutionz--kt-weight-pathzb[ktransformers parameter] The path of the quantized expert weights for amx kernel. A local folder.z --kt-methodAMXINT4zA[ktransformers parameter] Quantization formats for CPU execution.z --kt-cpuinferz9[ktransformers parameter] The number of CPUInfer threads.z--kt-threadpool-countrz^[ktransformers parameter] One-to-one with the number of NUMA nodes (one thread pool per NUMA).z--kt-num-gpu-expertsz4[ktransformers parameter] The number of GPU experts.z#--kt-max-deferred-experts-per-tokenz[ktransformers parameter] Maximum number of experts deferred to CPU per token. All MoE layers except the final one use this value; the final layer always uses 0.z--dllm-algorithmz3The diffusion LLM algorithm, such as LowConfidence.z--dllm-algorithm-configz@The diffusion LLM algorithm configurations. Must be a YAML file.z--enable-double-sparsityz Enable double sparsity attentionz--ds-channel-config-pathz.The path of the double sparsity channel configz--ds-heavy-channel-numz9The number of heavy channels in double sparsity attentionz--ds-heavy-token-numz7The number of heavy tokens in double sparsity attentionz--ds-heavy-channel-typez7The type of heavy channels in double sparsity attentionz--ds-sparse-decode-thresholdzThe minimum decode sequence length required before the double-sparsity backend switches from the dense fallback to the sparse decode kernel.z--cpu-offload-gbz2How many GBs of RAM to reserve for CPU offloading.z--offload-group-sizez)Number of layers per group in offloading.z--offload-num-in-groupz0Number of layers to be offloaded within a group.z--offload-prefetch-stepz Steps to prefetch in offloading.z--offload-modezMode of offloading.z--multi-item-scoring-delimiterzDelimiter token ID for multi-item scoring. Used to combine Query and Items into a single sequence: QueryItem1Item2... This enables efficient batch processing of multiple items against a single query.z--disable-radix-cachez*Disable RadixAttention for prefix caching.z--cuda-graph-max-bszjSet the maximum batch size for cuda graph. It will extend the cuda graph capture batch size to this value.z--cuda-graph-bsz+Set the list of batch sizes for cuda graph.)rUr`rVz--disable-cuda-graphzDisable cuda graph.z--disable-cuda-graph-paddingz\Disable cuda graph when padding is needed. Still uses cuda graph when padding is not needed.z--enable-profile-cuda-graphz'Enable profiling of cuda graph capture.z--enable-cudagraph-gczEnable garbage collection during CUDA graph capture. If disabled (default), GC is frozen during capture to speed up the process.z--enable-layerwise-nvtx-markerz:Enable layerwise NVTX profiling annotations for the model.z--enable-nccl-nvlsz;Enable NCCL NVLS for prefill heavy requests when available.z--enable-symm-memz2Enable NCCL symmetric memory for fast collectives.z.--disable-flashinfer-cutlass-moe-fp4-allgatherz?Disables quantize before all-gather for flashinfer cutlass moe.z--enable-tokenizer-batch-encodezEnable batch tokenization for improved performance when processing multiple text inputs. Do not use with image inputs, pre-tokenized input_ids, or input_embeds.z --disable-tokenizer-batch-decodez:Disable batch decoding when decoding multiple completions.z--disable-outlines-disk-cachezdDisable disk cache of outlines to avoid possible crashes related to file system or high concurrency.z--disable-custom-all-reducez;Disable the custom all-reduce kernel and fall back to NCCL.z--enable-mscclppzTEnable using mscclpp for small messages for all-reduce kernel and fall back to NCCL.z--enable-torch-symm-memzEnable using torch symm mem for all-reduce kernel and fall back to NCCL. Only supports CUDA device SM90 and above. SM90 supports world size 4, 6, 8. SM100 supports world size 6, 8.z--disable-overlap-schedulezVDisable the overlap scheduler, which overlaps the CPU scheduler with GPU model worker.z--enable-mixed-chunkzIEnabling mixing prefill and decode in a batch when using chunked prefill.z--enable-dp-attentionzEnabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported.z--enable-dp-lm-headzEnable vocabulary parallel across the attention TP group to avoid all-gather across DP groups, optimizing performance under DP attention.z--enable-two-batch-overlapz&Enabling two micro batches to overlap.z--enable-single-batch-overlapzALet computation and communication overlap within one micro batch.z"--tbo-token-distribution-thresholdzThe threshold of token distribution between two batches in micro-batch-overlap, determines whether to two-batch-overlap or two-chunk-overlap. Set to 0 denote disable two-chunk-overlap.z--enable-torch-compilez, fp8wo, fp8dq-per_tensor, fp8dq-per_rowz--enable-nan-detectionz0Enable the NaN detection for debugging purposes.z--enable-p2p-checkzPEnable P2P check for GPU access, otherwise the p2p access is allowed by default.z!--triton-attention-reduce-in-fp32zCast the intermediate attention results to fp32 to avoid possible crashes related to fp16.This only affects Triton attention kernels.z --triton-attention-num-kv-splitszThe number of KV splits in flash decoding Triton kernel. Larger value is better in longer context scenarios. The default value is 8.z"--triton-attention-split-tile-sizez\The size of split KV tile in flash decoding Triton kernel. Used for deterministic inference.z--num-continuous-decode-stepszRun multiple continuous decoding steps to reduce scheduling overhead. This can potentially increase throughput but may also increase time-to-first-token latency. The default value is 1, meaning only run one decoding step at a time.z--delete-ckpt-after-loadingz4Delete the model checkpoint after loading the model.z--enable-memory-saverzPAllow saving memory using release_memory_occupation and resume_memory_occupationz--enable-weights-cpu-backupzSave model weights (both main model and draft model, if any) to CPU memory during release_weights_occupation and resume_weights_occupationz!--enable-draft-weights-cpu-backupzfSave draft model weights to CPU memory during release_weights_occupation and resume_weights_occupationz--allow-auto-truncatezkAllow automatically truncating requests that exceed the maximum input length instead of returning an error.z--enable-custom-logit-processorz]Enable users to pass custom logit processors to the server (disabled by default for security)z--flashinfer-mla-disable-raggedzDisable shared experts fusion optimization for deepseek v3/r1.z--disable-chunked-prefix-cachezbDisable chunked prefix cache feature for deepseek, which should save overhead for short sequences.z--disable-fast-image-processorz;Adopt base image processor instead of fast image processor.z--keep-mm-feature-on-devicezLKeep multimodal feature tensors on device after processing to save D2H copy.z--enable-return-hidden-statesz.Enable returning hidden states with responses.z--enable-return-routed-expertsz=Enable returning routed experts of each layer with responses.z--scheduler-recv-intervalz\The interval to poll requests in scheduler. Can be set to >1 to reduce the overhead of this.z --numa-nodezUSets the numa node for the subprocesses. i-th element corresponds to i-th subprocess.z --enable-deterministic-inferencez=Enable deterministic inference mode with batch invariant ops.z--rl-on-policy-targetzBThe training system that SGLang needs to match for true on-policy.z --enable-attn-tp-input-scatteredzAllow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.z%--enable-nsa-prefill-context-parallelzTEnable context parallelism used in the long sequence prefill phase of DeepSeek v3.2.z--nsa-prefill-cp-modea&Token splitting mode for the prefill phase of DeepSeek v3.2 under context parallelism. Optional values: 'round-robin-split'(default), 'in-seq-split' 'round-robin-split' distributes tokens across ranks based on token_idx %% cp_size. It supports multi-batch prefill, fused MoE, and FP8 KV cache.z--enable-fused-qk-norm-ropez8Enable fused qk normalization and rope rotary embedding.z(--enable-precise-embedding-interpolationzEnable corner alignment for resize of embeddings grid to ensure more accurate(but slower) evaluation of interpolated embedding values.z --enable-dynamic-batch-tokenizerziEnable async dynamic batch tokenizer for improved performance when multiple requests arrive concurrently.z$--dynamic-batch-tokenizer-batch-sizezf[Only used if --enable-dynamic-batch-tokenizer is set] Maximum batch size for dynamic batch tokenizer.z'--dynamic-batch-tokenizer-batch-timeoutzm[Only used if --enable-dynamic-batch-tokenizer is set] Timeout in seconds for batching tokenization requests.z!--debug-tensor-dump-output-folderz&The output folder for dumping tensors.z--debug-tensor-dump-layersz8The layer ids to dump. Dump all layers if not specified.z--debug-tensor-dump-input-filez&The input filename for dumping tensorsz--debug-tensor-dump-injectz8Inject the outputs from jax as the input of every layer.z--disaggregation-moderTzOnly used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregatedz!--disaggregation-transfer-backendz=The backend for disaggregation transfer. Default is mooncake.z--disaggregation-bootstrap-portz=Bootstrap server port on the prefill server. Default is 8998.z--disaggregation-decode-tpzqDecode tp size. If not set, it matches the tp size of the current engine. This is only set on the prefill server.z--disaggregation-decode-dpzqDecode dp size. If not set, it matches the dp size of the current engine. This is only set on the prefill server.z--disaggregation-prefill-ppzWPrefill pp size. If not set, it is default to 1. This is only set on the decode server.z--disaggregation-ib-devicea&The InfiniBand devices for disaggregation transfer, accepts single device (e.g., --disaggregation-ib-device mlx5_0) or multiple comma-separated devices (e.g., --disaggregation-ib-device mlx5_0,mlx5_1). Default is None, which triggers automatic device detection when mooncake backend is enabled.z.--disaggregation-decode-enable-offload-kvcachez1 to reduce the overhead of this.z--encoder-onlyz7For MLLM with an encoder, launch an encoder-only serverz--language-onlyz2For VLM, load weights for the language model only.z--encoder-transfer-backendzMThe backend for encoder disaggregation transfer. Default is zmq_to_scheduler.z--encoder-urlszList of encoder server urls.)r`rUr3rVz--custom-weight-loaderzThe custom dataloader which used to update the model. Should be set with a valid import path, such as my_package.weight_load_funcz--weight-loader-disable-mmapz4Disable mmap while loading weight using safetensors.z0--remote-instance-weight-loader-seed-instance-ipzEThe ip of the seed instance for loading weights from remote instance.z:--remote-instance-weight-loader-seed-instance-service-portzOThe service port of the seed instance for loading weights from remote instance.z8--remote-instance-weight-loader-send-weights-group-portszGThe communication group ports for loading weights from remote instance.z'--remote-instance-weight-loader-backendr+rzlThe backend for loading weights from remote instance. Can be 'transfer_engine' or 'nccl'. Default is 'nccl'.z>--remote-instance-weight-loader-start-seed-via-transfer-enginezPStart seed server via transfer engine backend for remote instance weight loader.z--enable-pdmuxz6Enable PD-Multiplexing, PD running on greenctx stream.z--pdmux-config-pathz,The path of the PD-Multiplexing config file.z--sm-group-numzNumber of sm partition groups.--configzTRead CLI options from a config file. Must be a YAML file with configuration options.z--mm-max-concurrent-callsz6The max concurrent calls for async mm data processing.z--mm-per-request-timeoutz4The timeout for each multi-modal request in seconds.z$--enable-broadcast-mm-inputs-processz0Enable broadcast mm-inputs process in scheduler.z--mm-process-configzWMultimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`z--mm-enable-dp-encoderz_Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.z--limit-mm-data-per-requestz^Limit the number of multimodal inputs per request. e.g. '{"image": 1, "video": 1, "audio": 1}'z--decrypted-config-filez&The path of the decrypted config file.z--decrypted-draft-config-filez,The path of the decrypted draft config file.z--enable-prefix-mm-cachez@Enable prefix multimodal cache. Currently only supports mm-only.z--forward-hookszBJSON-formatted forward hook specifications to attach to the model.) add_argumentrrrrrrrrrrrrrrrrrrrr,rrrrrrrrrrrrrrrrrrrrrrrDeprecatedActionrrrrrrrBrCrrrrrrrrrrrrr r rrrrrrrrrMrNrr r!r"r#r$r%r&r(r)r/r0r1r2r4r5r6r7r9r DetectorMaprPr;r ToolCallParserEnumr<r?r@rArErFrGrIrJrKrrLoRAPathActionrNrQLORA_BACKEND_CHOICESrRrTrrUrWrVSAMPLING_BACKEND_CHOICESrXrrYrZr] NSA_CHOICESr^rr[rr\r_rdrerfrgrhrirjrlrmrrnMOE_A2A_BACKEND_CHOICESro,SPECULATIVE_DRAFT_MODEL_QUANTIZATION_CHOICESrprqrsrtrurwryrzr|r}r~rrrrrrrrrrrrrrrrr MAMBA_SCHEDULER_STRATEGY_CHOICESrrMAMBA_BACKEND_CHOICESr`rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrNSA_PREFILL_CP_SPLIT_CHOICESrrrrrrr rr r r r rrrr ENCODER_TRANSFER_BACKEND_CHOICESrrr+rrr!r"r#r$r'r&r(r)r*r%r+)rS bucket_ruletool_call_parser_choicesr3r3r add_cli_argsI sn                              zServerArgs.add_cli_argsargsargparse.Namespacecs`j_j_j_j_j_ j _ ddt |D}|difdd|DS)NcSsg|]}|jqSr3)namerattrr3r3rrsz,ServerArgs.from_cli_args..csi|]}|t|qSr3)rrrr3r r5z,ServerArgs.from_cli_args..r3)tensor_parallel_sizerpipeline_parallel_sizerattention_context_parallel_sizerBmoe_data_parallel_sizerCdata_parallel_sizer@expert_parallel_sizer| dataclassesfields)clsrattrsr3rr from_cli_argsszServerArgs.from_cli_argscCs2t|jrd|jd|jSd|jd|jS)Nzhttp://[z]:zhttp://:)r*rrr[r3r3rurls zServerArgs.urlcCs.ddlm}t|dr|jS|||_|jS)Nr) ModelConfigr)rrhasattrrfrom_server_args)rQrr3r3rrs   zServerArgs.get_model_configcCs,|jr|jn|j}|jr|jn|j}||fSr)rWrUrV)rQrrr3r3rrsz!ServerArgs.get_attention_backendscCs ddlm}|}|j|jkS)Nr) AttentionArch)rrrattention_archMLA)rQrrr3r3rrs  zServerArgs.use_mla_backendcCs|jduo|jduo|jduSr)rUrWrVr[r3r3rrs z'ServerArgs.is_attention_backend_not_setcCs |jdkS)Nr)rr[r3r3rrs z$ServerArgs.enable_mamba_extra_buffercCs tt|jSr)rrrr[r3r3rmamba_cache_chunk_sizes z!ServerArgs.mamba_cache_chunk_sizecCs~|j|j|jdksJd|jdkr#|jr|jdur|jr#Jd|jdkr4|jdkr4|js4Jd|jdks=Jd|j dksFJd|j dvsOJd d |j vs]Jd |j d | | |jdurq|jrqJd |jdkr|jdkr|j|jdksJd|jr|jdksJd|jdksJd|jdksJd|jsJdddl}tdkrtd|jd|jdksJd|d|j|d|j|jr|jdvsJd|jd|jdur|jsJd|jdksJd |jdksJd!|j d"krt!sJd#|j"s|j#rt$d$|j%r-|j&dur-t$d%|j'r;|j(d&kr=t$d'dSdS)(Nrz,tp_size must be divisible by number of nodesrzjPipeline parallelism is not compatible with overlap schedule, speculative decoding, mixed chunked prefill.z>multi-node data parallel is not supported unless dp attention!z base_gpu_id must be non-negativezgpu_id_step must be positive>Nrz3moe_dense_tp_size only support 1 and None currentlyrzserved_model_name cannot contain a colon (':') character. The colon is reserved for the 'model:adapter' syntax used in LoRA adapter specification. Invalid value: ''z7enable_mixed_chunk is required for speculative decodingrUz3chunked_prefill_size must be divisible by page_sizezQPD-Multiplexing is only supported with pipeline parallelism disabled (pp_size=1).rz7PD-Multiplexing is not compatible with chunked prefill.rz;PD-Multiplexing is not compatible with disaggregation mode.z8PD-Multiplexing is not compatible with overlap schedule.)rzxWARNING: PD-Multiplexing may experience performance degradation with torch versions > 2.6.x. Current torch version is z(. Please manually install torch 2.6.x.zTokenizer worker num must >= 1rdre)rr]zFTo use priority scheduling, schedule_policy must be 'fcfs' or 'lof'. 'z' is not supported.zMulti-item scoring requires radix cache to be disabled. Please set --disable-radix-cache when using --multi-item-scoring-delimiter.zMulti-item scoring requires chunked prefill to be disabled. Please set --chunked-prefill-size -1 when using --multi-item-scoring-delimiter.z.schedule_conservativeness must be non-negative mindsporez5MindSpore model impl is only supported on Ascend npu.zkPlease set --tokenizer-metrics-custom-labels-header when setting --tokenizer-metrics-allowed-custom-labels.zQ--export-metrics-to-file-dir is required when --export-metrics-to-file is enabledrszBWhen enabling two batch overlap, moe_a2a_backend cannot be 'none'.))rrrErrarr@rrrrr2check_lora_server_args%check_torch_2_9_1_cudnn_compatibilityrrrrrr.rYrZ __version__rvalidate_buckets_ruler$r%rrrrrrr$rrrXr.r/rr})rQrr3r3rcheck_server_args s              zServerArgs.check_server_argscCstdrdS|jrnddl}tdddkrpd}z|jj}Wn ty,d}Ynw|dur^t t |ddd}|dkr\d}d}d }||d |j d |d d |}t |dSd}d }t |d|dSdSdS)NSGLANG_DISABLE_CUDNN_CHECKrr|)rrrdgL"@zzzCRITICAL WARNING: PyTorch 2.9.1 & CuDNN Compatibility Issue Detected -------------------------------------------------------------------------------- Current Environment: PyTorch z | CuDNN z.2fac Issue: There is a KNOWN BUG in PyTorch 2.9.1's `nn.Conv3d` implementation when used with CuDNN versions older than 9.15. This can cause SEVERE PERFORMANCE DEGRADATION and EXCESSIVE MEMORY USAGE. Reference: https://github.com/pytorch/pytorch/issues/168167 Solution: You MUST upgrade CuDNN to version 9.15+ to ensure correctness. Run the following command immediately to fix: pip install nvidia-cudnn-cu12==9.16.0.29 Or you can disable this check by setting env var SGLANG_DISABLE_CUDNN_CHECK=1 -------------------------------------------------------------------------------- zqWARNING: Could not determine CuDNN version for torch==2.9.1. Please ensure CuDNN >= 9.15 to avoid nn.Conv3d bugs.)rrrrr.backendscudnnversionrGrrrr rYrZ)rQr cudnn_version version_floatREDBOLDRESETmsgr3r3rrsD     z0ServerArgs.check_torch_2_9_1_cudnn_compatibilitycCs|jdks Jd|jr$|jdurd|_tdn |jdur$td|jr}|jdur0d|_|jrI|jd}|jdurB|j|ksIJd||jd vrRtd t |jt r|j}g|_|D]W}t |t rd |vr{| d d \}}t ||dd }n6t ||dd }n.t |trd|vrd|vsJd|t |d|d|ddd }n tdt|d|j|q`n&t |jtrdd|jD|_n|jdurg|_n tdt|jd|jrt|j|_d|jvrt|jd ksJdtt|_|jdkrtd|jd|jd|js&|jr"|js&Jd|jdurV|j|jks?Jd|jd |jt|j|jksVJd!|jd"t|j|jdurd#|jkrid$krynJd%|j|jd @dksJd%dSdSdS)&Nrz$max_loras_per_batch must be positiveTz>--enable-lora is set to True because --lora-paths is provided.FzG--enable-lora is set to False, any provided lora_paths will be ignored.rzEnabling LoRA overlap loading requires pinning LoRA adapter weights in CPU memory, so --max-loaded-loras must be less than or equal to double --max-loras-per-batch: )r~NzBCurrently LoRA is only compatible with NGRAM speculative decoding.=r lora_name lora_pathpinnedrrzmWhen providing LoRA paths as a list of dict, each dict should contain 'lora_name' and 'lora_path' keys. Got: rz,Invalid type for item in --lora-paths list: z$. Expected a string or a dictionary.cSsg|] \}}t||ddqS)Frr)rkvr3r3rrs z5ServerArgs.check_lora_server_args..zInvalid type for --lora-paths: z". Expected a list or a dictionary.allzWIf 'all' is specified in --lora-target-modules, it should be the only module specified.rizLoRA backend 'csgmv' does not yet support embedding or lm_head layers; dropping 'embed_tokens' and 'lm_head' from --lora-target-modules=all. To apply LoRA to these, use --lora-backend triton. embed_tokenslm_headzWhen no initial --lora-paths is provided, you need to specify both --max-lora-rank and --lora-target-modules for LoRA initialization.zZmax_loaded_loras should be greater than or equal to max_loras_per_batch. max_loaded_loras=z, max_loras_per_batch=zNThe number of LoRA paths should not exceed max_loaded_loras. max_loaded_loras=z , lora_paths=rSrz>--max-lora-chunk-size must be a power of 2 between 16 and 128.)rPrMrIrYrZrJrNrarXrLrrr6rrOrbrUappenditemsrLrrrrRdiscardrKrT)rQmax_loaded_loras_limitrMrrrnlora_refr3r3rrs                     dz!ServerArgs.check_lora_server_args prefill_tp decode_tpcCs8t||}t||}||dksJd|d|dS)NrzTDifferent tp size is supported only when one tp is multiple of the other. decode_tp=z , prefill_tp=)rr)rQrr larger_tp smaller_tpr3r3rr-3s  z"ServerArgs.validate_disagg_tp_sizearg_name buckets_rulec Cs|sdSt|dksJ|d|d}|dvs$Jd|d|d|dkrt|dks9J|d t|zt|d }t|d }t|d }Wnttfy^ J|dw|d kslJ|d||dksyJ|d||dksJ|d|dS|dkrt|d ksJ|dt|dS|dkrt|d ksJ|dz dd|d dD}WntyJ|dwtt|t|ksJ|dtdd|DsJ|ddSdS)Nrz cannot be empty list)tser3customz Unsupported z rule type: 'z-'. Must be one of: 'tse', 'default', 'custom'rrvzK TSE rule requires exactly 4 parameters: ['tse', middle, base, count], got rrr|FzP TSE rule parameters must be: ['tse', , , ]z& TSE base must be larger than 1, got: z" TSE count must be positive, got: z# TSE middle must be positive, got: r3z? default rule should only have one parameter: ['default'], got rzH custom rule requires at least one bucket value: ['custom', value1, ...]cSsg|]}t|qSr3)r)rrfr3r3rrasz4ServerArgs.validate_buckets_rule..z* custom rule bucket values must be numericz8 custom rule bucket values should not contain duplicatescss|]}|dkVqdS)rNr3)rvalr3r3rrgs z3ServerArgs.validate_buckets_rule..z1 custom rule bucket values should be non-negative)rrrrX IndexErrorrr)rQrrrulemiddlebasecount bucket_valuesr3r3rr;sb       z ServerArgs.validate_buckets_rulecCst|jdd}|dur dS|j}d}t|dd}t|dd}d}d}||d} ||d} | dkr6| | nd } d } d | | d } td td | } || }|||_dS) N vision_configgffffff?num_hidden_layersrw hidden_sizer{rrrg?rg?)rrrrr)rQrr original_server_arg_mem_fraction!base_mem_fraction_reduction_ratiovit_num_layersvit_hidden_sizebaseline_vit_layersbaseline_vit_hidden_sizecurrent_complexity_scorebaseline_complexity_scorecomplexity_ratiosensitivity_scaledynamic_adjustment_factorfinal_overall_factorr3r3rrks,    z&ServerArgs.adjust_mem_fraction_for_vlmcCs6tjddurtddS|jrtddSdS)Nzmooncake.enginezqFailed to import mooncake.engine. Does not support using TransferEngine as remote instance weight loader backend.FzMemory saver is enabled, which is not compatible with TransferEngine. Does not support using TransferEngine as remote instance weight loader backend.T) importlibutil find_specrYrZrr[r3r3rr,sz#ServerArgs.validate_transfer_enginecCs&|jrdS|jdkr|jdkrdSdS)NTrBr+F)rrrr[r3r3r1remote_instance_weight_loader_use_transfer_engines   zr@rArBrDrCrEr=rFrGrHrIrrJrKrLrMrNrPrO staticmethodr classmethodrrrrrrrpropertyrrrrr-rrr,rr3r3r3rrsX                                                                                                                                                                                       ` ) ) CMO25 4-,/ [7    +y0&rzOptional[ServerArgs]_global_server_args server_argscCs|adSr)r)rr3r3r$set_global_server_args_for_schedulersrrcCstdurtdtS)Nz"Global server args is not set yet!)rrXr3r3r3rget_global_server_argssrargvrcCsLt}t|d|vrddlm}||}||}||}t|S)a# Prepare the server arguments from the command line arguments. Args: args: The command line arguments. Typically, it should be `sys.argv[1:]` to ensure compatibility with `parse_args` when no arguments are passed. Returns: The server arguments. rr)ConfigArgumentMerger) argparseArgumentParserrr$sglang.srt.server_args_config_parserrmerge_config_with_args parse_argsr)rrSr config_mergerraw_argsr3r3rprepare_server_argss     r c@sZeZdZUded<ded<ded<ded<ded<ded<d ed <e ddddZd S)PortArgsrtokenizer_ipc_namescheduler_input_ipc_namedetokenizer_ipc_namerr rpc_ipc_namemetrics_ipc_namertokenizer_worker_ipc_nameNrrdp_rankr worker_portsrrcCsZ|jdur t}n|j}|jdkrd}n dtjddj}|jsTtdtjddjdtjddjdtjddj|dtjddjdtjddj|dS|jdkrf|j durfd|j t f}n|j drzt |j \}}|t|f}n|j d}t|d ksJd |\}} t| } | d} | d} | d } | d } |dur| d }n |dusJ||}z1|durt| d t| dt| dt|dt| dt| d|dus|durt|dWntytd| d| d| d|d| wtd|d| d|d|d|d| |d|d| d|d| |dS)Nrzipc://F)delete)r r r rrrrr[rrz9please provide --dist-init-addr as host:port of head noder|rvdist_init_port port_basedetokenizer_portrrpc_portrscheduler_input_portz'Port is already in use. dist_init_port=z port_base=z detokenizer_port=z nccl_port=z scheduler_input_port=ztcp://)rrrtempfileNamedTemporaryFilerrr rErDrZMQ_TCP_PORT_DELTAr*rrr6rrr/rXrY exception)rrrrrrDport_numrdist_init_hostrrrrrrr3r3rinit_news|               zPortArgs.init_new)NN)rrrrrrrr )rrrrrr r3r3r3rr s r c@seZdZdddZdS)rNcCsg}|rAt|ts Jd|D]1}|}|dr;|dr;t|}d|vr,d|vs5Jt|d||q||qt ||j |dS)NzExpected a list of LoRA paths.{}rrzQ looks like a JSON str, but it does not contain 'lora_name' and 'lora_path' keys.) rLrr/r*endswithrMrNreprrsetattrrn)rQrS namespacevalues option_stringrMrobjr3r3r__call__Js    zLoRAPathAction.__call__r)rrrr*r3r3r3rrIsrmessagercCstd|ddS)Nzr)rYrZ)r+r3r3rprint_deprecated_warning]sr,cs(eZdZdfdd ZdddZZS) rrc s"tt|j||fd|i|dS)Nr`)superr__init__)rQoption_stringsrnr`kwargs __class__r3rr.bs  zDeprecatedAction.__init__NcCstd|ddS)NzThe command line argument 'z7' is deprecated and will be removed in future versions.)r,)rQrSr&r'r(r3r3rr*gs zDeprecatedAction.__call__)rr)rrrr.r* __classcell__r3r3r1rrasrrQcCsJ|j}|jd}|jdkrdS|dvrdS|dvrdS|dvr#dSdS)z Automatically choose the parameters for speculative decoding. You can tune them on your own models and prompts with scripts/playground/bench_speculative.py rr})r|rrv)LlamaForCausalLM)rvrO) r$rrErrrrr%r&rrrr)Grok1ForCausalLMGrok1VForCausalLM)rrrra)rQrarchr3r3rr)ms   r))rr)rr)rrrr)r+r)rQr)~r __future__rrrrimportlib.utilrMloggingrmrkrtypingrrrrrrr sglang.srt.connectorr sglang.srt.environr -sglang.srt.function_call.function_call_parserr -sglang.srt.layers.attention.fla.chunk_delta_hr rsglang.srt.lora.lora_registryr"sglang.srt.parser.reasoning_parserrsglang.srt.utils.commonrrrrrrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0&sglang.srt.utils.hf_transformers_utilsr1 sglang.utilsr2 getLoggerrrYrrrrrrrrrrrrrDEFAULT_LORA_EVICTION_POLICYrrrrrrrrrrrrrrrrrrrrrrr dataclassrrrr$set_global_server_args_for_tokenizerrrr!DP_ATTENTION_HANDSHAKE_PORT_DELTAr Actionrr,rr)r3r3r3rs $       "        D   i