o -i=@sUddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z m Z e rdZ eed<dZedBed<eZeed<dZeed <d Zeed <dZedBed <dZedBed <dZeed<dZeed<dZedBed<d Zeed<dZeed<dZedBed<dZeed<dZ edBed<dZ!edBed<dZ"edBed<dZ#edBed<ej$%dZ&eed<ej$%dZ'eed<dZ(eed <dZ)eed!<dZ*eed"<dZ+eed#<d$Z,eed%<d&Z-eed'<dZ.eed(<d)Z/eed*<dZ0edBed+<d,Z1eed-<dZ2eed.<d/Z3e4ed0<dZ5eed1<dZ6edBed2<dZ7edBed3<dZ8edBed4<dZ9eed5<dZ:edBed6<dZ;eed7<ej$eed:<d;Z?eed<<d$Z@eed=<d,ZAe d>ed?<dZBeed@<d$ZCeedA<dZDeedB<dCZEe dDedE<ej$d?d]d@d?d`ed`d^gd_ddAdadBd?dbdCd?dcdDd?dddEd?dedFd?dhedhdgdgdidGd?ddHd?ddId?ddJd?deddKd?d dLd?d dMd?idNdOd?d dPd?d dQd?ddRd?ddSd?ddTd?ddUd?d{ed|dVd?d}dWd?d~dXd?ddYd?ddZd?dd[d?dd\d?dd]d?dd^d?idd_d?dd`d?ddad?d dbd?d!dcd?d"ddd?d#ded?d%dfd?d+dgd?d'dhd?d*did?d(djd?d-dkd?d.dld?d0dmd?d1dnd?d2dod?id3dpd?d4dqd?d5drd?d6dsd?d7dtd?d?ed?d,gd>d@dud?dAdvd?dEedEdCdwdCgdGdxd?dHdyd?dJdzd?dLd{d?dNd|d?dOd}d?dQd~d?dSdd?idUdd?dWdd?dYedYdXgdddAd9dd?d:dd?dBdd?d<dd?d=dd?dfdd?djdd?ddd?ddd?dldd?dmdd?dndd?dodd?dpdd?idqdd?drdd?dsdd?dudd?dvdd?dtdd?dydd?dzdd?dwdd?dxdd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?iddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ideddgdddd?ddd?ddd?ddd?ddd?ddd?ddd?deddd?ddd?ddd?ddd?ddd?ddd?ddd?ddÐd?iddĐd?ddŐd?ddƐd?ddǐd?ddȐd?ddɐd?ddʐd?dːd̐d?dd͐d?ddΐd?ddϐd?dedddАdgddҐd?ddӐd?ddԐd?ddՐd?dd֐d?iddאd?ddؐd?ddِd?ddڐd?deddgdâdŐdېd?dƐdܐd?dǐdݐd?dȐdސd?dɐdߐd?ddd?ddd?ddd?dϐdd?dАdd?dѐdd?dӐdd?idՐdd?dאdd?deddgd٢deddgdˢdΐdd?dܐdd?ddd?ddd?dݐdd?dސdd?ddd?ddd?deddddgddd?ddd?ddd?ddd?iddd?ddd?ddd?ddd?ddd?deddgdddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?dedggd d dd d?idd d?dd d?ddd?ddd?deddd?d dd?d dd?d dd?d dd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?ddd?dd?eddddgdd?d d?d!d?d"d?d#d?d$Zeeege ffed%<d&efd'd(ad&efd)d*Zd6d+d,Zd6d-d.Zd/d0Z d&efd1d2Z d&eee ffd3d4Z dS(7N)Callable) TYPE_CHECKINGAnyLiteral VLLM_HOST_IP VLLM_PORTVLLM_RPC_BASE_PATHFVLLM_USE_MODELSCOPE< VLLM_RINGBUFFER_WARNING_INTERVALVLLM_NCCL_SO_PATHLD_LIBRARY_PATHVLLM_ROCM_SLEEP_MEM_CHUNK_SIZE LOCAL_RANKCUDA_VISIBLE_DEVICESVLLM_ENGINE_ITERATION_TIMEOUT_SiXVLLM_ENGINE_READY_TIMEOUT_S VLLM_API_KEY"VLLM_DEBUG_LOG_API_SERVER_RESPONSES3_ACCESS_KEY_IDS3_SECRET_ACCESS_KEYS3_ENDPOINT_URLVLLM_MODEL_REDIRECT_PATHz ~/.cache/vllmVLLM_CACHE_ROOTz~/.config/vllmVLLM_CONFIG_ROOThttps://stats.vllm.aiVLLM_USAGE_STATS_SERVERVLLM_NO_USAGE_STATSVLLM_DO_NOT_TRACKVLLM_USAGE_SOURCETVLLM_CONFIGURE_LOGGINGINFOVLLM_LOGGING_LEVELVLLM_LOGGING_PREFIXext://sys.stdoutVLLM_LOGGING_STREAMVLLM_LOGGING_CONFIG_PATHautoVLLM_LOGGING_COLORNO_COLOR$@VLLM_LOG_STATS_INTERVALVLLM_TRACE_FUNCTIONVLLM_USE_FLASHINFER_SAMPLERVLLM_PP_LAYER_PARTITIONVLLM_CPU_KVCACHE_SPACEVLLM_CPU_OMP_THREADS_BINDVLLM_CPU_NUM_OF_RESERVED_CPUVLLM_CPU_SGL_KERNEL xla_cacheVLLM_XLA_CACHE_PATHVLLM_XLA_CHECK_RECOMPILATION@VLLM_FUSED_MOE_CHUNK_SIZE)VLLM_ENABLE_FUSED_MOE_ACTIVATION_CHUNKING)r)ncclshm&VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE&VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMMVLLM_USE_RAY_WRAPPED_PP_COMMVLLM_XLA_USE_SPMDfork)rAspawnVLLM_WORKER_MULTIPROC_METHODassetsVLLM_ASSETS_CACHEVLLM_ASSETS_CACHE_MODEL_CLEANVLLM_IMAGE_FETCH_TIMEOUTVLLM_VIDEO_FETCH_TIMEOUT VLLM_AUDIO_FETCH_TIMEOUTVLLM_MEDIA_URL_ALLOW_REDIRECTSVLLM_MEDIA_LOADING_THREAD_COUNTVLLM_MAX_AUDIO_CLIP_FILESIZE_MBopencvVLLM_VIDEO_LOADER_BACKENDhttpVLLM_MEDIA_CONNECTORblake3VLLM_MM_HASHER_ALGORITHMcudaVLLM_TARGET_DEVICE12.9VLLM_MAIN_CUDA_VERSIONhighest)r\highmediumVLLM_FLOAT32_MATMUL_PRECISIONMAX_JOBS NVCC_THREADSVLLM_USE_PRECOMPILED$VLLM_SKIP_PRECOMPILED_VERSION_SUFFIXVLLM_DOCKER_BUILD_CONTEXTVLLM_KEEP_ALIVE_ON_ENGINE_DEATH)DebugReleaseRelWithDebInfoCMAKE_BUILD_TYPEVERBOSEVLLM_ALLOW_LONG_MAX_MODEL_LENi'VLLM_RPC_TIMEOUTVLLM_HTTP_TIMEOUT_KEEP_ALIVE VLLM_PLUGINSVLLM_LORA_RESOLVER_CACHE_DIRVLLM_TORCH_CUDA_PROFILEVLLM_TORCH_PROFILER_DIR!VLLM_TORCH_PROFILER_RECORD_SHAPES'VLLM_TORCH_PROFILER_WITH_PROFILE_MEMORY%VLLM_TORCH_PROFILER_DISABLE_ASYNC_LLMVLLM_TORCH_PROFILER_WITH_STACKVLLM_TORCH_PROFILER_WITH_FLOPSVLLM_TORCH_PROFILER_USE_GZIP(VLLM_TORCH_PROFILER_DUMP_CUDA_TIME_TOTALVLLM_PROFILER_DELAY_ITERSVLLM_PROFILER_MAX_ITERSVLLM_USE_AOT_COMPILEVLLM_USE_BYTECODE_HOOKVLLM_FORCE_AOT_LOADVLLM_USE_MEGA_AOT_ARTIFACTVLLM_USE_TRITON_AWQ VLLM_ALLOW_RUNTIME_LORA_UPDATINGVLLM_SKIP_P2P_CHECKVLLM_DISABLED_KERNELSVLLM_DISABLE_PYNCCLVLLM_ROCM_USE_AITERVLLM_ROCM_USE_AITER_PAGED_ATTNVLLM_ROCM_USE_AITER_LINEARVLLM_ROCM_USE_AITER_MOEVLLM_ROCM_USE_AITER_RMSNORMVLLM_ROCM_USE_AITER_MLAVLLM_ROCM_USE_AITER_MHA VLLM_ROCM_USE_AITER_FP4_ASM_GEMMVLLM_ROCM_USE_AITER_TRITON_ROPEVLLM_ROCM_USE_AITER_FP8BMMVLLM_ROCM_USE_AITER_FP4BMM%VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION)VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTSVLLM_ROCM_USE_AITER_TRITON_GEMMVLLM_ROCM_USE_SKINNY_GEMMVLLM_ROCM_FP8_PADDINGVLLM_ROCM_MOE_PADDINGVLLM_ROCM_CUSTOM_PAGED_ATTN!VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUTVLLM_ENABLE_V1_MULTIPROCESSINGVLLM_LOG_BATCHSIZE_INTERVALVLLM_DISABLE_COMPILE_CACHEQ_SCALE_CONSTANTK_SCALE_CONSTANTdV_SCALE_CONSTANTVLLM_SERVER_DEV_MODEVLLM_V1_OUTPUT_PROC_CHUNK_SIZEVLLM_MLA_DISABLEg?VLLM_RAY_PER_WORKER_GPUSVLLM_RAY_BUNDLE_INDICESVLLM_CUDART_SO_PATH VLLM_DP_RANKVLLM_DP_RANK_LOCAL VLLM_DP_SIZEVLLM_USE_STANDALONE_COMPILEVLLM_DP_MASTER_IPVLLM_DP_MASTER_PORTVLLM_MOE_DP_CHUNK_SIZEVLLM_ENABLE_MOE_DP_CHUNKVLLM_RANDOMIZE_DP_DUMMY_INPUTSstrict)rfillspanVLLM_RAY_DP_PACK_STRATEGYVLLM_MARLIN_USE_ATOMIC_ADD)int8fp8VLLM_MARLIN_INPUT_DTYPEVLLM_MXFP4_USE_MARLINVLLM_DEEPEPLL_NVFP4_DISPATCHVLLM_V1_USE_OUTLINES_CACHEVLLM_TPU_BUCKET_PADDING_GAPVLLM_TPU_MOST_MODEL_LENVLLM_TPU_USING_PATHWAYSVLLM_USE_DEEP_GEMMVLLM_MOE_USE_DEEP_GEMMVLLM_USE_DEEP_GEMM_E8M0%VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALESrelax)skipfullrVLLM_DEEP_GEMM_WARMUPVLLM_USE_FUSED_MOE_GROUPED_TOPK#VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFERVLLM_USE_FLASHINFER_MOE_FP16VLLM_USE_FLASHINFER_MOE_FP8VLLM_USE_FLASHINFER_MOE_FP4latency) throughputr masked_gemmVLLM_FLASHINFER_MOE_BACKEND%VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZEVLLM_XGRAMMAR_CACHE_MB VLLM_MSGPACK_ZERO_COPY_THRESHOLD!VLLM_ALLOW_INSECURE_SERIALIZATION localhostVLLM_NIXL_SIDE_CHANNEL_HOSTiVLLM_NIXL_SIDE_CHANNEL_PORTi&#VLLM_MOONCAKE_BOOTSTRAP_PORTallgather_reducescatter)naivepplxdeepep_high_throughputdeepep_low_latencymorirflashinfer_all2allvVLLM_ALL2ALL_BACKENDi"VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE%VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDSVLLM_SLEEP_WHEN_IDLEVLLM_MQ_MAX_CHUNK_BYTES_MBi,"VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS)NHDHNDVLLM_KV_CACHE_LAYOUTVLLM_COMPUTE_NANS_IN_LOGITSVLLM_USE_NVFP4_CT_EMULATIONSNONE)FPINT8INT6INT4r#VLLM_ROCM_QUICK_REDUCE_QUANTIZATION(VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16(VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MBiVLLM_NIXL_ABORT_REQUEST_TIMEOUTVLLM_MORIIO_CONNECTOR_READ_MODEVLLM_MORIIO_QP_PER_TRANSFERVLLM_MORIIO_POST_BATCH_SIZEVLLM_MORIIO_NUM_WORKERS#VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUTVLLM_ENABLE_CUDAGRAPH_GCVLLM_LOOPBACK_IP2VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHEVLLM_ENABLE_RESPONSES_API_STOREVLLM_NVFP4_GEMM_BACKENDVLLM_HAS_FLASHINFER_CUBIN#VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16VLLM_ROCM_FP8_MFMA_PAGE_ATTN+VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASSVLLM_ALLREDUCE_USE_SYMM_MEMVLLM_TUNED_CONFIG_FOLDER#VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS$VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT(VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS$VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY VLLM_CUSTOM_SCOPES_FOR_PROFILINGVLLM_NVTX_SCOPES_FOR_PROFILING#VLLM_KV_EVENTS_USE_INT_BLOCK_HASHESVLLM_OBJECT_STORAGE_SHM_BUFFER#VLLM_OBJECT_STORAGE_SHM_BUFFER_NAMEiVLLM_DEEPEP_BUFFER_SIZE_MB,VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE!VLLM_DEEPEP_LOW_LATENCY_USE_MNNVLVLLM_DBO_COMM_SMSVLLM_PATTERN_MATCH_DEBUGVLLM_DEBUG_DUMP_PATH!VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE.VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNINGVLLM_USE_NCCL_SYMM_MEMVLLM_NCCL_INCLUDE_PATHVLLM_USE_FBGEMM VLLM_GC_DEBUGVLLM_DEBUG_WORKSPACE"VLLM_DISABLE_SHARED_EXPERTS_STREAM*VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLDbinary)runpackedVLLM_COMPILE_CACHE_SAVE_FORMATVLLM_USE_V2_MODEL_RUNNERVLLM_LOG_MODEL_INSPECTIONVLLM_DEBUG_MFU_METRICSVLLM_DISABLE_LOG_LOGOVLLM_LORA_DISABLE_PDLcCtdtjtjddS)NXDG_CACHE_HOME~z.cacheosgetenvpathjoin expanduserr0r0F/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/envs.pyget_default_cache_rootr2cCr')NXDG_CONFIG_HOMEr)z.configr*r0r0r0r1get_default_config_rootr3r5valuereturncCs|durdSt|SN)intr6r0r0r1maybe_convert_intsr;cCs|durdStt|Sr8)boolr9r:r0r0r1maybe_convert_bools r=cCtttddS)Nr0r<r9r+r,r0r0r0r1disable_compile_cacherAcCs\ddlm}ddlm}ddlm}|drts|sdnd}| o-tj d|dkS) Nr)vllm_is_batch_invariant)current_platform)is_torch_equal_or_newerz 2.10.0.dev1r?r{) *vllm.model_executor.layers.batch_invariantrCvllm.platformsrDvllm.utils.torch_utilsrErAis_cpur+environget)rCrDrE default_valuer0r0r1use_aot_compiles    rNenv_namedefaultchoicescase_sensitivecs dtdBffdd }|S)a Create a lambda that validates environment variable against allowed choices Args: env_name: Name of the environment variable default: Default value if not set (can be None) choices: List of valid string options or callable that returns list case_sensitive: Whether validation should be case sensitive Returns: Lambda function for environment_variables dict r7Ncstt}|dur Strn}s"|}dd|D}n|}|}||vr8td|dd|d|S)NcSg|]}|qSr0lower.0choicer0r0r1 Sz@env_with_choices.._get_validated_env..Invalid value 'z' for . Valid options: .)r+r,callablerU ValueError)r6actual_choices check_value check_choicesrRrQrPrOr0r1_get_validated_envIs z,env_with_choices.._get_validated_envstr)rOrPrQrRrdr0rcr1env_with_choices6srgc dttffdd }|S)a Create a lambda that validates environment variable containing comma-separated values against allowed choices Args: env_name: Name of the environment variable default: Default list of values if not set choices: List of valid string options or callable that returns list case_sensitive: Whether validation should be case sensitive Returns: Lambda function for environment_variables dict that returns list of strings r7c st}|dur Sdd|dD}|sStr n}|D]&}s4|}dd|D}n|}|}||vrJtd|dd|dq$|S) NcSsg|] }|r|qSr0)strip)rWvr0r0r1rY~szJenv_list_with_choices.._get_validated_env_list..,cSrSr0rTrVr0r0r1rYrZr[z' in r\r])r+r,splitr^rUr_)r6valuesr`valrarbrcr0r1_get_validated_env_listxs* z6env_list_with_choices.._get_validated_env_list)listrf)rOrPrQrRror0rcr1env_list_with_choicescsrqcrh)z Creates a lambda which that validates environment variable containing comma-separated values against allowed choices which returns choices as a set. r7csttSr8)setrqr0rcr0r1_get_validated_env_setrBz4env_set_with_choices.._get_validated_env_set)rrrf)rOrPrQrRrsr0rcr1env_set_with_choicess rtc Cs|dtjvrdStdd}zt|WSty=}zddlm}||}|jr0td|ddtd|d|d}~ww) zGet the port from VLLM_PORT environment variable. Returns: The port number as an integer if VLLM_PORT is set, None otherwise. Raises: ValueError: If VLLM_PORT is a URI, suggest k8s service discovery issue. rNr?r) parse_urlz VLLM_PORT 'z' appears to be a URI. This may be caused by a Kubernetes service discovery issue,check the warning in: https://docs.vllm.ai/en/stable/serving/env_vars.htmlz' must be a valid integer)r+rKr,r9r_ urllib3.utilruscheme)porterrruparsedr0r0r1 get_vllm_ports"     r{default_factorycsdtffdd }|S)z Create a lambda that returns an environment variable value if set, or generates and sets a default value using the provided factory function. r7cs*t}|dur |S}|tj<|Sr8)r+r,rK)r6rMr|rOr0r1_get_or_set_defaults  z3get_env_or_set_default.._get_or_set_defaultre)rOr|r~r0r}r1get_env_or_set_defaults  rcCtddS)NrYrXr+r,rUr0r0r0r1rcCstddp dS)Nr[rrZrr0r0r0r1rs)rRcC tddS)Nr`r+r,r0r0r0r1r cCr)Nrarr0r0r0r1rrcCs*tjdddvpttjdS)NrbrrFtrueVLLM_PRECOMPILED_WHEEL_LOCATION)r+rKrLrirUr<r0r0r0r1rs cCtttjddS)Nrcr?r<r9r+rKrLr0r0r0r1rcCtjdddvS)Nrdrrr+rKrLrirUr0r0r0r1rs cCr>)Nrjr?r@r0r0r0r1rrZc C tjtdtjtdS)Nrvllm)r+r-r/r,r.r5r0r0r0r1r c Cr)Nrrr+r-r/r,r.r2r0r0r0r1r"rcC tddS)Nrrrr0r0r0r1r,rcCstdtS)Nr )r+r,tempfile gettempdirr0r0r0r1r4scCtjdddkS)Nr Falserr+rKrLrUr0r0r0r1r9 cCttjddS)Nr 60r9r+rKrLr0r0r0r1r>  CUDA_HOMEcCtjddS)Nrr+rKrLr0r0r0r1rCcCr)Nr rr0r0r0r1rFrcCr)Nrrr0r0r0r1rIrcCr)Nr256rr0r0r0r1rKrcCstjdddkS)NrrFrr0r0r0r1rQ cCr)Nrrr0r0r0r1rWcCr)Nrrr0r0r0r1r\rcCr)Nr|rFrr0r0r0r1rcrcCtjdddkS)Nr}r?rFrr0r0r0r1ricCr)Nr~r?rFrr0r0r0r1rmrcCr)Nrr?rr0r0r0r1rsrcCr)Nrrr0r0r0r1rurcCr)Nrrrr0r0r0r1rwrcCr)Nr600rr0r0r0r1r|rcCr)Nrrr0r0r0r1rrcCr)Nrrrrr0r0r0r1rrcCr)Nrrr0r0r0r1rrcCr)Nrrr0r0r0r1rrcCr)Nrrr0r0r0r1rrcCtjddS)Nrrrr0r0r0r1rrcCr)Nrr?rFrr0r0r0r1rrcCs$tjddptjddpddkS)Nr DO_NOT_TRACKr?rFrr0r0r0r1rs  cCr)Nr! productionrr0r0r0r1rrcCr>)Nr"rFr@r0r0r0r1rcC tdS)Nr(rr0r0r0r1r cCr)Nr$r#)r+r,upperr0r0r0r1rrcCr)Nr'r&rr0r0r0r1rrcCr)Nr%rrr0r0r0r1rrcCr)Nr*r)rr0r0r0r1rrcCstdddkS)Nr+r?rr0r0r0r1rrcCs ttdd}dkr|SdS)Nr-z10.gr,floatr+r,)rnr0r0r1rscCttddS)Nr.r?r9r+r,r0r0r0r1rrcCs dtjvrtttjdSdS)Nr/)r+rKr<r9r0r0r0r1rs  cCr)Nr0rr0r0r0r1rrcCdtjvr ttddSdS)Nr1r?r+rKr9r,r0r0r0r1rs cCr)Nr2r)rr0r0r0r1rrcCr)Nr3r?rr0r0r0r1rs  cCr>)Nr4r?r@r0r0r0r1rrZcCr>)Nr>r?r@r0r0r0r1rrcCr>)Nr?rFr@r0r0r0r1rrrBc C"tjtdtjtddS)NrErrDrr0r0r0r1r cCr>)NrFr?r@r0r0r0r1rrcCr)NrH5rr0r0r0r1rrcCr)NrJ30rr0r0r0r1r cCr)NrL10rr0r0r0r1rrcCr>)NrMrFr@r0r0r0r1rrcCr)NrO8rr0r0r0r1rrcCr)NrQ25rr0r0r0r1r rcCr)NrSrRrr0r0r0r1rcCr)NrUrTrr0r0r0r1r!r)rVsha256sha512c Cr)Nr6rr5rr0r0r0r1r/rcCr>)Nr7r?r@r0r0r0r1r6rcCr>)Nr@r?r@r0r0r0r1r:rZcCttdtdS)Nr9r8r9r+r,rfr0r0r0r1r;rcCr>)Nr:rFr@r0r0r0r1rArcCr>)Nrer?r@r0r0r0r1rFrcCr)Nrkr?rrr0r0r0r1rMVLLM_TEST_FORCE_FP8_MARLINcCr)Nrr?rrr0r0r0r1rSrVLLM_TEST_FORCE_LOAD_FORMATcCr)Nrdummyrr0r0r0r1rWrcCr)Nrl10000rr0r0r0r1r\rcCr)Nrmrrr0r0r0r1r^rcCsdtjvrdStjddS)Nrnrkr+rKrlr0r0r0r1rd cCr)Nrorr0r0r0r1rjrcCr)Nrprr0r0r0r1rorcCr)Nrqrr0r0r0r1rrrcCr)Nrrrr0r0r0r1ru cCr)Nrsrr0r0r0r1rzrcCr)Nrurr0r0r0r1rrcCr)Nrvrr0r0r0r1rrcCr)Nrtrr0r0r0r1rrcCr)Nryrr0r0r0r1rrcCr)Nrzrr0r0r0r1rrcCr)Nrwrr0r0r0r1rrcCr)Nrxrr0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrZcCr)Nrr?rrr0r0r0r1rrcCstdddkS)NrrFrr0r0r0r1rrcCsdtjvrgStjddS)Nrrkrr0r0r0r1rrcCtdddvS)NrrrrFrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)NrTruerrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rcCr)Nrrrrr0r0r0r1rrcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1rrBcCr>)NrrFr@r0r0r0r1rrZcCr>)NrrFr@r0r0r0r1rrZcCr)Nrrrrr0r0r0r1rrBcCr)Nrrrrr0r0r0r1r rBcCr)Nrrrrr0r0r0r1rrcCttjddS)Nrr;r+rKrLr0r0r0r1r"rcCr)Nr200rr0r0r0r1r&rcCr)Nrrrr0r0r0r1r(rcCr)Nr100rr0r0r0r1r*rcCr>)NrrFr@r0r0r0r1r,rcCr)Nr-1rr0r0r0r1r/rcCr>)Nrr?r@r0r0r0r1r6rZcCr)Nr128rr0r0r0r1r>rcCr>)Nrr?r@r0r0r0r1rBrZcCr)Nrz1.0rr0r0r0r1rGrcCr)Nrrrr0r0r0r1rMrcCr)Nrrr0r0r0r1rPrcCr)Nrr?rr0r0r0r1rRrcCsttdtjtjS)Nr)r9r+r,sysmodules__name__rr0r0r0r1rUcCr)NrrFrr0r0r0r1rYrcCr)Nrz 127.0.0.1rr0r0r0r1r[rcCr)Nrr?rr0r0r0r1r]rcCr)Nrrrr0r0r0r1rcrcCr>)NrrFr@r0r0r0r1rdrcCr)Nrr?rFrr0r0r0r1rhrcCr)Nrrrr0r0r0r1rwrVLLM_CI_USE_S3cCr)Nrr?rFrr0r0r0r1r{rcCr)Nrrr0r0r0r1rrcCr)Nrr?rFrr0r0r0r1rrcCr)Nr)r=r+rKrLr0r0r0r1rrrrcCr>)Nrr?r@r0r0r0r1rrcCr)Nrr?rFrr0r0r0r1rrcCsdtjvr ttjdSdS)Nrr)r+rKr9r0r0r0r1rs cCr)Nrrr0r0r0r1rrcCstdtddvS)Nproxy JAX_PLATFORMSr)r<r+r,rUr0r0r0r1rrcCr>)NrrFr@r0r0r0r1rrZcCr>)NrrFr@r0r0r0r1rrcCr>)NrrFr@r0r0r0r1rrcCr>)NrrFr@r0r0r0r1rrcCr>)NrrFr@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr)Nr512rr0r0r0r1rrcCr)Nrrrr0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr)Nrrrr0r0r0r1r rcCr)Nr5600rr0r0r0r1rrcCr)Nr8998rr0r0r0r1rrcCr)Nrrrr0r0r0r1r:rcCr)Nr163840rr0r0r0r1rAr.VLLM_FLASHINFER_ALLREDUCE_FUSION_THRESHOLDS_MBcCsttddS)Nrz{})jsonloadsr+r,r0r0r0r1rJs $VLLM_MOE_ROUTING_SIMULATION_STRATEGYcCstjddS)Nrrrr0r0r0r1rSscCr)NrrFrr0r0r0r1rWrcCr>)Nrr?r@r0r0r0r1r\rZcCr)Nr16rr0r0r0r1r`rcCr)Nr300rr0r0r0r1rerrrcCr>)Nrr?r@r0r0r0r1rurcCr>)Nrr?r@r0r0r0r1r{rcCr)Nr480rr0r0r0r1rrcCr)Nrrrrr0r0r0r1rrBcCr)NrrFrr0r0r0r1rrcCr)Nrrrr0r0r0r1rrcCr)NrrFrr0r0r0r1rrcCr)Nrrrr0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rr)zflashinfer-cudnnzflashinfer-trtllmzflashinfer-cutlasscutlassmarlincCr>)Nrr?r@r0r0r0r1rrcCr)Nrrrr0r0r0r1rrVLLM_PROCESS_NAME_PREFIXcCr)NrVLLMrr0r0r0r1rrcCr>)NrrFr@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr>)NrrFr@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1rrcCr)Nrrr0r0r0r1rr) containercode_interpreterweb_search_preview)rPrQcCr>)Nrr?r@r0r0r0r1rrcCr>)Nr r?r@r0r0r0r1rrcCr>)Nr r?r@r0r0r0r1rrcCr>)Nr r?r@r0r0r0r1rrcCr>)Nr rFr@r0r0r0r1rrcCsdtjS)NVLLM_OBJECT_STORAGE_SHM_BUFFER_)uuiduuid4hexr0r0r0r1rrcCr)Nr1024rr0r0r0r1rrcCr>)Nrr?r@r0r0r0r1r rcCr>)Nrr?r@r0r0r0r1rrcCr)Nr20rr0r0r0r1rrcCr>)NrrFr@r0r0r0r1rrcCr>)NrrFr@r0r0r0r1rrcCr>)Nrr?r@r0r0r0r1r#rcCr)Nrrr0r0r0r1r'rcCr>)Nrr?r@r0r0r0r1r)rZcCr)Nrrrr0r0r0r1r/rcCr>)Nrr?r@r0r0r0r1r2rZcCr>)Nrr?r@r0r0r0r1r4rcCstttddS)Nrrrr0r0r0r1r;rr cCr>)Nr"r?r@r0r0r0r1rHrcCr>)Nr#r?r@r0r0r0r1rNrcCr>)Nr$r?r@r0r0r0r1rRrcCr>)Nr%r?r@r0r0r0r1rVrZcCr>)Nr&r?r@r0r0r0r1rYrZ)rr!r"r#r$r%r&environment_variablesnamecCs&|tvr t|Stdtd|)z Gets environment variables lazily. NOTE: After enable_envs_cache() invocation (which triggered after service initialization), all environment variables will be cached. module  has no attribute )rAttributeErrorrrr0r0r1 __getattr__`s rcCs ttdS)z6Checked if __getattr__ is wrapped with functools.cache cache_clear)hasattrrr0r0r0r1_is_envs_cache_enabledls rcCs*trdSttatD]}t|q dS)at Enables caching of environment variables. This is useful for performance reasons, as it avoids the need to re-evaluate environment variables on every call. NOTE: Currently, it's invoked after service initialization to reduce runtime overhead. This also means that environment variables should NOT be updated after the service is initialized. N)r functoolscacherr)keyr0r0r1enable_envs_cachers    rcCstrtjadSdS)zr Resets the environment variables cache. It could be used to isolate environments between unit tests. N)rr __wrapped__r0r0r0r1disable_envs_caches rcCs ttSr8)rprkeysr0r0r0r1__dir__s rcCs&|tvr |tjvStdtd|)z3Check if an environment variable is explicitly set.rr)rr+rKrrrr0r0r1is_sets rc Cshd}ddlm}i}tD].\}}||vrqz|}Wnty7}z td||WYd}~qd}~ww||||<qgd}|D] }|t|||<qE|S)zReturn env vars used for torch.compile cache keys. Start with every known vLLM env var; drop entries in `ignored_factors`; hash everything else. This keeps the cache key aligned across workers.>:r`r+rrrrrrrrrEr rr*r$r r4rr}r%r'rr rrVLLM_CPU_MOE_PREPACKrrUrrr1r-rrLrHr(rrrJr2rSrr3rmrCrFrrMrrrrerQrOr rrrr)normalize_valuezBSkipping environment variable %s while hashing compile factors: %sN) +RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES+RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES*RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES0RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES-RAY_EXPERIMENTAL_NOSET_HABANA_VISIBLE_MODULES.RAY_EXPERIMENTAL_NOSET_NEURON_RT_VISIBLE_CORES(RAY_EXPERIMENTAL_NOSET_TPU_VISIBLE_CHIPS-RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR.RAY_EXPERIMENTAL_NOSET_RBLN_RT_VISIBLE_DEVICES) vllm.config.utilsrritems Exceptionloggerwarningr+r,) ignored_factorsrfactorsfactorgetterrawexcray_noset_env_varsvarr0r0r1compile_factorss, =  r)T)r7N( rrloggingr+rrrcollections.abcrtypingrrrrrf__annotations__rr9rr r r<r r rrrrrrrrrrrrr-r/rrrrr r!r"r$r%r'r(r*r+r-rr.r/r0r1r2r3r4r.r6r7r9r:r=r>r?r@rCrErFrHrJrLrMrOrQrSrUrWrYr[r_r`rarbrcrdrerirjrkrlrmrnrprorprqrrrsrtrurvrwrxryrzr{r|r}r~rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr r r r rrrrrrrrrrrrrrrrr!r"r#r$r%r&r2r5r;r=rArNrgrqrtr{r getLoggerrrrdictrrrrrrobjectrr0r0r0r1s                                                                                                                                                ""1;          #*  . 4 = GL O T Y^ a d f lrw{ ~         # $ % ' * + 1 6 9 ; = ? B D G M O U X ] ` fn  s y~        # ) 4 >D L S W X ^ c jpt y {        " ' - 0 3 7 ; = G L P U Z ` e i n s x }        " $ (. 7 A E G I K NQ U ] a f l o q t x z |     " & +/ 5 ; A F J N P T Xf p u y }      ! ' + / 3AU [ bkt x }     $ ( , 0 4 6 ;D R VZ b n r vz ~ $ )/49=BFJLRUW ^      (  &