import os import subprocess import warnings from contextlib import ExitStack, contextmanager from enum import IntEnum from typing import Any @contextmanager def temp_set_env(*, allow_sglang: bool = False, **env_vars: Any): """Temporarily set environment variables, restoring originals on exit. By default, SGLANG_*/SGL_* keys are rejected — use ``Envs`` descriptors for those. Pass ``allow_sglang=True`` only for special env vars that intentionally bypass ``environ.py`` (e.g. ``SGLANG_DUMPER_*``). """ if not allow_sglang: for key in env_vars: if key.startswith("SGLANG_") or key.startswith("SGL_"): raise ValueError("temp_set_env should not be used for sglang env vars") backup = {key: os.environ.get(key) for key in env_vars} try: for key, value in env_vars.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = str(value) yield finally: for key, value in backup.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value class EnvField: _allow_set_name = True def __init__(self, default: Any): self.default = default # NOTE: environ can only accept str values, so we need a flag to indicate # whether the env var is explicitly set to None. self._set_to_none = False def __set_name__(self, owner, name): assert EnvField._allow_set_name, "Usage like `a = envs.A` is not allowed" self.name = name def parse(self, value: str) -> Any: raise NotImplementedError() def get(self) -> Any: value = os.getenv(self.name) # Explicitly set to None if self._set_to_none: assert value == str(None) return None # Not set, return default if value is None: return self.default try: return self.parse(value) except ValueError as e: warnings.warn( f'Invalid value for {self.name}: {e}, using default "{self.default}"' ) return self.default def is_set(self): return self.name in os.environ def set(self, value: Any): self._set_to_none = value is None os.environ[self.name] = str(value) @contextmanager def override(self, value: Any): backup_present = self.name in os.environ backup_value = os.environ.get(self.name) backup_set_to_none = self._set_to_none self.set(value) yield if backup_present: os.environ[self.name] = backup_value else: os.environ.pop(self.name, None) self._set_to_none = backup_set_to_none def clear(self): os.environ.pop(self.name, None) self._set_to_none = False def __bool__(self): raise RuntimeError( "Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`" ) def __len__(self): raise RuntimeError( "Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`" ) class EnvTuple(EnvField): def parse(self, value: str) -> tuple[str, ...]: return tuple(s.strip() for s in value.split(",") if s.strip()) class EnvStr(EnvField): def parse(self, value: str) -> str: return value class EnvBool(EnvField): def parse(self, value: str) -> bool: value = value.lower() if value in ["true", "1", "yes", "y"]: return True if value in ["false", "0", "no", "n"]: return False raise ValueError(f'"{value}" is not a valid boolean value') class EnvInt(EnvField): def parse(self, value: str) -> int: try: return int(value) except ValueError: raise ValueError(f'"{value}" is not a valid integer value') class EnvFloat(EnvField): def parse(self, value: str) -> float: try: return float(value) except ValueError: raise ValueError(f'"{value}" is not a valid float value') class ToolStrictLevel(IntEnum): """ Defines the strictness levels for tool call parsing and validation. OFF: No strict validation FUNCTION: Enables structural tag constraints for all tools PARAMETER: Enforces strict parameter validation for all tools """ OFF = 0 FUNCTION = 1 PARAMETER = 2 class Envs: # fmt: off # Model & File Download SGLANG_USE_MODELSCOPE = EnvBool(False) SGLANG_DISABLED_MODEL_ARCHS = EnvTuple(tuple()) # Logging Options SGLANG_LOG_GC = EnvBool(False) SGLANG_LOG_FORWARD_ITERS = EnvBool(False) SGLANG_LOG_MS = EnvBool(False) SGLANG_DISABLE_REQUEST_LOGGING = EnvBool(False) SGLANG_LOG_REQUEST_EXCEEDED_MS = EnvInt(-1) SGLANG_LOG_REQUEST_HEADERS = EnvTuple(tuple()) SGLANG_LOG_SCHEDULER_STATUS_TARGET = EnvStr("") SGLANG_LOG_SCHEDULER_STATUS_INTERVAL = EnvFloat(60.0) # SGLang CI SGLANG_IS_IN_CI = EnvBool(False) SGLANG_IS_IN_CI_AMD = EnvBool(False) SGLANG_CUDA_COREDUMP = EnvBool(False) SGLANG_CUDA_COREDUMP_DIR = EnvStr("/tmp/sglang_cuda_coredumps") SGLANG_TEST_MAX_RETRY = EnvInt(None) # Constrained Decoding (Grammar) SGLANG_GRAMMAR_POLL_INTERVAL = EnvFloat(0.005) SGLANG_GRAMMAR_MAX_POLL_ITERATIONS = EnvInt(10000) SGLANG_DISABLE_OUTLINES_DISK_CACHE = EnvBool(False) # CuTe DSL GDN Decode SGLANG_USE_CUTEDSL_GDN_DECODE = EnvBool(False) # Test & Debug SGLANG_DETECT_SLOW_RANK = EnvBool(False) SGLANG_TEST_STUCK_DETOKENIZER = EnvFloat(0) SGLANG_TEST_STUCK_DP_CONTROLLER = EnvFloat(0) SGLANG_TEST_STUCK_SCHEDULER_INIT = EnvFloat(0) SGLANG_TEST_STUCK_TOKENIZER = EnvFloat(0) SGLANG_TEST_CRASH_AFTER_STREAM_OUTPUTS = EnvInt(0) IS_BLACKWELL = EnvBool(False) IS_H200 = EnvBool(False) SGLANG_SET_CPU_AFFINITY = EnvBool(False) SGLANG_PROFILE_WITH_STACK = EnvBool(True) SGLANG_PROFILE_RECORD_SHAPES = EnvBool(True) SGLANG_PROFILE_V2 = EnvBool(False) SGLANG_RECORD_STEP_TIME = EnvBool(False) SGLANG_FORCE_SHUTDOWN = EnvBool(False) SGLANG_DEBUG_MEMORY_POOL = EnvBool(False) SGLANG_TEST_REQUEST_TIME_STATS = EnvBool(False) SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(False) SGLANG_SIMULATE_ACC_LEN = EnvFloat(-1) SGLANG_SIMULATE_ACC_METHOD = EnvStr("multinomial") SGLANG_TORCH_PROFILER_DIR = EnvStr("/tmp") SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS = EnvInt(500) SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE = EnvInt(64) SGLANG_NATIVE_MOVE_KV_CACHE = EnvBool(False) SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(True) # Scheduler: memory leak test SGLANG_TEST_RETRACT = EnvBool(False) SGLANG_TEST_RETRACT_INTERVAL = EnvInt(3) SGLANG_TEST_RETRACT_NO_PREFILL_BS = EnvInt(2 ** 31) SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY = EnvInt(0) SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_IDLE = EnvBool(True) # Scheduler: new token ratio hyperparameters SGLANG_INIT_NEW_TOKEN_RATIO = EnvFloat(0.7) SGLANG_MIN_NEW_TOKEN_RATIO_FACTOR = EnvFloat(0.14) SGLANG_NEW_TOKEN_RATIO_DECAY_STEPS = EnvInt(600) SGLANG_RETRACT_DECODE_STEPS = EnvInt(20) SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION = EnvInt(4096) # Scheduler: recv interval SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DEFAULT = EnvInt(1000) SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DECODE = EnvInt(1) SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_TARGET_VERIFY = EnvInt(1) SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_NONE = EnvInt(1) # PD Disaggregation (runtime) # NOTE: For SGLANG_DISAGGREGATION_THREAD_POOL_SIZE, the effective default is # computed dynamically at runtime based on cpu_count; see disaggregation backends. SGLANG_DISAGGREGATION_THREAD_POOL_SIZE = EnvInt(None) SGLANG_DISAGGREGATION_QUEUE_SIZE = EnvInt(4) SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT = EnvInt(300) SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL = EnvFloat(5.0) SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE = EnvInt(2) SGLANG_DISAGGREGATION_WAITING_TIMEOUT = EnvInt(300) SGLANG_DISAGGREGATION_NIXL_BACKEND = EnvStr("UCX") # Scheduler: others: SGLANG_EMPTY_CACHE_INTERVAL = EnvFloat(-1) # in seconds. Set if you observe high memory accumulation over a long serving period. SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP = EnvBool(False) SGLANG_SCHEDULER_MAX_RECV_PER_POLL = EnvInt(-1) SGLANG_EXPERIMENTAL_CPP_RADIX_TREE = EnvBool(False) SGLANG_DYNAMIC_CHUNKING_SMOOTH_FACTOR = EnvFloat(0.75) SGLANG_SCHEDULER_SKIP_ALL_GATHER = EnvBool(False) SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE = EnvBool(False) SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(30) SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None) SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1) SGLANG_REQ_WAITING_TIMEOUT = EnvFloat(-1) # in seconds SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False) SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds # Test: pd-disaggregation SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr("mooncake") SGLANG_TEST_PD_DISAGG_DEVICES = EnvStr(None) # Model Parallel SGLANG_USE_MESSAGE_QUEUE_BROADCASTER = EnvBool(True) SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS = EnvBool(False) # Override the distributed init method used by torch.distributed.init_process_group. # Set to "env://" to use an externally-created TCPStore via MASTER_ADDR/MASTER_PORT. SGLANG_DISTRIBUTED_INIT_METHOD_OVERRIDE = EnvStr(None) # Tool Calling SGLANG_FORWARD_UNKNOWN_TOOLS = EnvBool(False) # Hi-Cache SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None) SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None) SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR = EnvStr(None) # Mooncake KV Transfer SGLANG_MOONCAKE_CUSTOM_MEM_POOL = EnvStr(None) ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE = EnvBool(False) ASCEND_NPU_PHY_ID = EnvInt(-1) SGLANG_MOONCAKE_SEND_AUX_TCP = EnvBool(False) # Mooncake Store SGLANG_HICACHE_MOONCAKE_CONFIG_PATH = EnvStr(None) MOONCAKE_MASTER = EnvStr(None) MOONCAKE_CLIENT = EnvStr(None) MOONCAKE_LOCAL_HOSTNAME = EnvStr("localhost") MOONCAKE_TE_META_DATA_SERVER = EnvStr("P2PHANDSHAKE") MOONCAKE_GLOBAL_SEGMENT_SIZE = EnvStr("4gb") MOONCAKE_PROTOCOL = EnvStr("tcp") MOONCAKE_DEVICE = EnvStr("") MOONCAKE_MASTER_METRICS_PORT = EnvInt(9003) MOONCAKE_CHECK_SERVER = EnvBool(False) MOONCAKE_STANDALONE_STORAGE = EnvBool(False) # AMD & ROCm SGLANG_USE_AITER = EnvBool(False) SGLANG_ROCM_FUSED_DECODE_MLA = EnvBool(False) SGLANG_ROCM_DISABLE_LINEARQUANT = EnvBool(False) # NPU SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT = EnvBool(False) SGLANG_NPU_USE_MULTI_STREAM = EnvBool(False) SGLANG_NPU_USE_MLAPO = EnvBool(False) # Forward native implementation for activation gelu tanh for model Skywork-Reward-Gemma-2-27B-v0.2 SGLANG_NPU_FORWARD_NATIVE_GELUTANH = EnvBool(False) # Forward native implementation for gemma rms norm for model Skywork-Reward-Gemma-2-27B-v0.2 SGLANG_NPU_FORWARD_NATIVE_GEMMA_RMS_NORM = EnvBool(False) # Quantization SGLANG_INT4_WEIGHT = EnvBool(False) SGLANG_CPU_QUANTIZATION = EnvBool(False) SGLANG_USE_DYNAMIC_MXFP4_LINEAR = EnvBool(False) SGLANG_FORCE_FP8_MARLIN = EnvBool(False) SGLANG_MOE_NVFP4_DISPATCH = EnvBool(False) SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN = EnvBool(False) SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2 = EnvBool(False) SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE = EnvBool(False) # Flashinfer SGLANG_IS_FLASHINFER_AVAILABLE = EnvBool(True) SGLANG_ENABLE_FLASHINFER_FP8_GEMM = EnvBool(False) # Default to the pick from flashinfer SGLANG_FLASHINFER_FP4_GEMM_BACKEND = EnvStr("") SGLANG_FLASHINFER_WORKSPACE_SIZE = EnvInt(384 * 1024 * 1024) # Triton SGLANG_TRITON_DECODE_ATTN_STATIC_KV_SPLITS = EnvBool(False) SGLANG_USE_CUSTOM_TRITON_KERNEL_CACHE = EnvBool(False) # Torch Compile SGLANG_ENABLE_TORCH_COMPILE = EnvBool(False) # EPLB SGLANG_EXPERT_LOCATION_UPDATER_LOG_INPUT = EnvBool(False) SGLANG_EXPERT_LOCATION_UPDATER_CANARY = EnvBool(False) SGLANG_EXPERT_LOCATION_UPDATER_LOG_METRICS = EnvBool(False) SGLANG_LOG_EXPERT_LOCATION_METADATA = EnvBool(False) SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR = EnvStr("/tmp") SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL = EnvInt(0) SGLANG_ENABLE_EPLB_BALANCEDNESS_METRIC = EnvBool(False) # TBO SGLANG_TBO_DEBUG = EnvBool(False) # DeepGemm SGLANG_ENABLE_JIT_DEEPGEMM = EnvBool(True) SGLANG_JIT_DEEPGEMM_PRECOMPILE = EnvBool(True) SGLANG_JIT_DEEPGEMM_FAST_WARMUP = EnvBool(False) SGLANG_JIT_DEEPGEMM_COMPILE_WORKERS = EnvInt(4) SGLANG_IN_DEEPGEMM_PRECOMPILE_STAGE = EnvBool(False) SGLANG_DG_CACHE_DIR = EnvStr(os.path.expanduser("~/.cache/deep_gemm")) SGLANG_DG_USE_NVRTC = EnvBool(False) SGLANG_USE_DEEPGEMM_BMM = EnvBool(False) # DeepSeek MHA Optimization SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD = EnvInt(8192) # DeepEP SGLANG_DEEPEP_BF16_DISPATCH = EnvBool(False) SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128) SGLANG_DEEPEP_LL_COMBINE_SEND_NUM_SMS = EnvInt(32) SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO = EnvBool(False) # NSA Backend SGLANG_NSA_FUSE_TOPK = EnvBool(True) SGLANG_NSA_ENABLE_MTP_PRECOMPUTE_METADATA = EnvBool(True) SGLANG_USE_FUSED_METADATA_COPY = EnvBool(True) SGLANG_VERIFY_FUSED_METADATA_COPY = EnvBool(False) SGLANG_NSA_FORCE_MLA = EnvBool(False) # sgl-kernel SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK = EnvBool(False) # vLLM dependencies (TODO: they have been deprecated, we can remove them safely) USE_VLLM_CUTLASS_W8A8_FP8_KERNEL = EnvBool(False) USE_TRITON_W8A8_FP8_KERNEL = EnvBool(False) SGLANG_RETURN_ORIGINAL_LOGPROB = EnvBool(False) SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN = EnvBool(False) SGLANG_MOE_PADDING = EnvBool(False) SGLANG_CUTLASS_MOE = EnvBool(False) HF_HUB_DISABLE_XET = EnvBool(False) DISABLE_OPENAPI_DOC = EnvBool(False) SGLANG_ENABLE_TORCH_INFERENCE_MODE = EnvBool(False) SGLANG_IS_FIRST_RANK_ON_NODE = EnvBool(True) SGLANG_SUPPORT_CUTLASS_BLOCK_FP8 = EnvBool(False) SGLANG_SYNC_TOKEN_IDS_ACROSS_TP = EnvBool(False) SGLANG_ENABLE_COLOCATED_BATCH_GEN = EnvBool(False) # Deterministic inference SGLANG_ENABLE_DETERMINISTIC_INFERENCE = EnvBool(False) # Use 1-stage all-reduce kernel on AMD (deterministic, fixed accumulation order) # If not set: auto (enabled when --enable-deterministic-inference is on) # Set to 1: force enable (even without --enable-deterministic-inference) # Set to 0: force disable (use default Aiter AR even with --enable-deterministic-inference) SGLANG_USE_1STAGE_ALLREDUCE = EnvBool(False) SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE = EnvInt(4096) SGLANG_FLASHINFER_DECODE_SPLIT_TILE_SIZE = EnvInt(2048) SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE = EnvInt(4096) SGLANG_TRITON_DECODE_SPLIT_TILE_SIZE = EnvInt(256) # RoPE cache configuration SGLANG_SPEC_EXPANSION_SAFETY_FACTOR = EnvInt(2) SGLANG_ROPE_CACHE_SAFETY_MARGIN = EnvInt(256) SGLANG_ROPE_CACHE_ALIGN = EnvInt(128) # Overlap Spec V2 SGLANG_ENABLE_SPEC_V2 = EnvBool(False) SGLANG_ENABLE_OVERLAP_PLAN_STREAM = EnvBool(False) # Spec Config SGLANG_SPEC_ENABLE_STRICT_FILTER_CHECK = EnvBool(True) # VLM SGLANG_VLM_CACHE_SIZE_MB = EnvInt(100) SGLANG_IMAGE_MAX_PIXELS = EnvInt(16384 * 28 * 28) SGLANG_RESIZE_RESAMPLE = EnvStr("") SGLANG_MM_BUFFER_SIZE_MB = EnvInt(0) SGLANG_MM_PRECOMPUTE_HASH = EnvBool(False) SGLANG_VIT_ENABLE_CUDA_GRAPH = EnvBool(False) SGLANG_MM_SKIP_COMPUTE_HASH = EnvBool(False) # VLM Item CUDA IPC Transport SGLANG_USE_CUDA_IPC_TRANSPORT = EnvBool(False) SGLANG_MM_FEATURE_CACHE_MB = EnvInt(4 * 1024) SGLANG_MM_ITEM_MEM_POOL_RECYCLE_INTERVAL_SEC = EnvFloat(0.05) # MM splitting behavior control SGLANG_ENABLE_MM_SPLITTING = EnvBool(False) # Mamba SGLANG_MAMBA_CONV_DTYPE = EnvStr("bfloat16") SGLANG_MAMBA_SSM_DTYPE = EnvStr(None) # Release & Resume Memory SGLANG_MEMORY_SAVER_CUDA_GRAPH = EnvBool(False) # Sparse Embeddings SGLANG_EMBEDDINGS_SPARSE_HEAD = EnvStr(None) # Logits processor SGLANG_ENABLE_LOGITS_PROCESSER_CHUNK = EnvBool(False) SGLANG_LOGITS_PROCESSER_CHUNK_SIZE = EnvInt(2048) # Tool-Call behavior SGLANG_TOOL_STRICT_LEVEL = EnvInt(ToolStrictLevel.OFF) # Ngram SGLANG_NGRAM_FORCE_GREEDY_VERIFY = EnvBool(False) # Warmup SGLANG_WARMUP_TIMEOUT = EnvFloat(-1) # in seconds. If a warmup forward batch takes longer than this, the server will crash to prevent hanging. Recommend to increase warmup timeout to 1800 to accommodate some kernel JIT precache e.g. deep gemm # Health Check SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True) # External models SGLANG_EXTERNAL_MODEL_PACKAGE = EnvStr("") SGLANG_EXTERNAL_MM_MODEL_ARCH = EnvStr("") SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE = EnvStr("") # Numa SGLANG_NUMA_BIND_V2 = EnvBool(True) # Metrics SGLANG_ENABLE_METRICS_DEVICE_TIMER = EnvBool(False) SGLANG_ENABLE_METRICS_DP_ATTENTION = EnvBool(False) # Tokenizer SGLANG_PATCH_TOKENIZER = EnvBool(False) # TODO enable by default # TokenizerManager SGLANG_REQUEST_STATE_WAIT_TIMEOUT = EnvInt(4) # Symmetric Memory SGLANG_SYMM_MEM_PREALLOC_GB_SIZE = EnvInt(-1) # Aiter SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False) # fmt: on # EPD SGLANG_ENCODER_RECV_TIMEOUT = EnvFloat(180.0) SGLANG_ENCODER_SEND_TIMEOUT = EnvFloat(180.0) envs = Envs() EnvField._allow_set_name = False def _print_deprecated_env(new_name: str, old_name: str): if old_name in os.environ: warnings.warn( f"Environment variable {old_name} will be deprecated, please use {new_name} instead" ) os.environ[new_name] = os.environ[old_name] def _warn_deprecated_env_to_cli_flag(env_name: str, suggestion: str): """Warn when a deprecated environment variable is used. This is for env vars that are deprecated in favor of CLI flags. """ if env_name in os.environ: warnings.warn(f"Environment variable {env_name} is deprecated. {suggestion}") def _convert_SGL_to_SGLANG(): _print_deprecated_env("SGLANG_LOG_GC", "SGLANG_GC_LOG") _print_deprecated_env( "SGLANG_ENABLE_FLASHINFER_FP8_GEMM", "SGLANG_ENABLE_FLASHINFER_GEMM" ) _print_deprecated_env( "SGLANG_MOE_NVFP4_DISPATCH", "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH" ) _print_deprecated_env( "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK", "SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK", ) _deprecated_ms_to_s = { "SGLANG_QUEUED_TIMEOUT_MS": "SGLANG_REQ_WAITING_TIMEOUT", "SGLANG_FORWARD_TIMEOUT_MS": "SGLANG_REQ_RUNNING_TIMEOUT", } for old_name, new_name in _deprecated_ms_to_s.items(): if old_name in os.environ: ms_val = os.environ[old_name] warnings.warn( f"Environment variable {old_name} (in ms) is deprecated, " f"please use {new_name} (in seconds) instead" ) os.environ[new_name] = str(float(ms_val) / 1000.0) for key, value in os.environ.items(): if key.startswith("SGL_"): new_key = key.replace("SGL_", "SGLANG_", 1) warnings.warn( f"Environment variable {key} is deprecated, please use {new_key}" ) os.environ[new_key] = value _convert_SGL_to_SGLANG() _warn_deprecated_env_to_cli_flag( "SGLANG_ENABLE_FLASHINFER_FP8_GEMM", "It will be completely removed in 0.5.7. Please use '--fp8-gemm-backend=flashinfer_trtllm' instead.", ) _warn_deprecated_env_to_cli_flag( "SGLANG_ENABLE_FLASHINFER_GEMM", "It will be completely removed in 0.5.7. Please use '--fp8-gemm-backend=flashinfer_trtllm' instead.", ) _warn_deprecated_env_to_cli_flag( "SGLANG_SUPPORT_CUTLASS_BLOCK_FP8", "It will be completely removed in 0.5.7. Please use '--fp8-gemm-backend=cutlass' instead.", ) _warn_deprecated_env_to_cli_flag( "SGLANG_FLASHINFER_FP4_GEMM_BACKEND", "It will be completely removed in 0.5.9. Please use '--fp4-gemm-backend' instead.", ) _warn_deprecated_env_to_cli_flag( "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE", "Please use '--enable-prefill-delayer' instead.", ) _warn_deprecated_env_to_cli_flag( "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES", "Please use '--prefill-delayer-max-delay-passes' instead.", ) _warn_deprecated_env_to_cli_flag( "SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK", "Please use '--prefill-delayer-token-usage-low-watermark' instead.", ) # Import cuda_coredump to trigger auto-injection of CUDA env vars # when SGLANG_CUDA_COREDUMP=1. Best-effort; for strict guarantees, # set CUDA_* env vars in the shell before launching Python. import sglang.srt.debug_utils.cuda_coredump # noqa: F401, E402 def example_with_exit_stack(): # Use this style of context manager in unit test exit_stack = ExitStack() exit_stack.enter_context(envs.SGLANG_TEST_RETRACT.override(False)) assert envs.SGLANG_TEST_RETRACT.get() is False exit_stack.close() assert envs.SGLANG_TEST_RETRACT.get() is None def example_with_subprocess(): command = ["python", "-c", "import os; print(os.getenv('SGLANG_TEST_RETRACT'))"] with envs.SGLANG_TEST_RETRACT.override(True): process = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) process.wait() output = process.stdout.read().decode("utf-8").strip() assert output == "True" process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = process.stdout.read().decode("utf-8").strip() assert output == "None" def example_with_implicit_bool_avoidance(): @contextmanager def assert_throws(message_matcher: str): try: yield except Exception as e: assert message_matcher in str(e), f"{e=}" print(f"assert_throws find expected error: {e}") return raise AssertionError(f"assert_throws do not see exceptions") with assert_throws("Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`"): if envs.SGLANG_TEST_RETRACT: pass with assert_throws("Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`"): if (1 != 1) or envs.SGLANG_TEST_RETRACT: pass with assert_throws("Please use `envs.YOUR_FLAG.get()` instead of `envs.YOUR_FLAG`"): if envs.SGLANG_TEST_RETRACT or (1 == 1): pass def examples(): # Example usage for envs envs.SGLANG_TEST_RETRACT.clear() assert envs.SGLANG_TEST_RETRACT.get() is False envs.SGLANG_TEST_RETRACT.set(None) assert envs.SGLANG_TEST_RETRACT.is_set() and envs.SGLANG_TEST_RETRACT.get() is None envs.SGLANG_TEST_RETRACT.clear() assert not envs.SGLANG_TEST_RETRACT.is_set() envs.SGLANG_TEST_RETRACT.set(True) assert envs.SGLANG_TEST_RETRACT.get() is True with envs.SGLANG_TEST_RETRACT.override(None): assert ( envs.SGLANG_TEST_RETRACT.is_set() and envs.SGLANG_TEST_RETRACT.get() is None ) assert envs.SGLANG_TEST_RETRACT.get() is True envs.SGLANG_TEST_RETRACT.set(None) with envs.SGLANG_TEST_RETRACT.override(True): assert envs.SGLANG_TEST_RETRACT.get() is True assert envs.SGLANG_TEST_RETRACT.is_set() and envs.SGLANG_TEST_RETRACT.get() is None example_with_exit_stack() example_with_subprocess() example_with_implicit_bool_avoidance() if __name__ == "__main__": examples()