o پi@sUdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z mZmZmZmZmZmZmZeedddddlZddlZddlZddlZddlZdd lmZmZmZm Z dd l!m"Z"dd l#m$Z$dd l%m&Z&m'Z'm(Z(dd l)m*Z*m+Z+ddl,m-Z-m.Z.ddl/m0Z0ddl1m2Z2m3Z3m4Z4m5Z5ddl6m7Z7m8Z8m9Z9ddl:m;Z;ddlZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHddlImJZJddlKmLZLddlMmNZNddlOmPZPddlQmRZRddlSmTZTddlUmVZVmWZWddlXmYZYddlZm[Z[ddl\m]Z]ddl^m_Z_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~ddlmZmZmZmZmZmZdd lmZdd!lmZmZdd"lmZdd#lmZdd$lmZdd%lmZmZdd&lmZmZdd'lmZmZmZmZmZmZdd(lmZmZmZdd)lmZdd*lmZeeZeeeed+d,Zeed-d.ZejGd/d0d0Zdaeeed1<d2efd3d4Zd5efd6d7Zd5efd8d9Ze d:efd;d<Zeeed=rdnd>d?Zeje$d@gdAd@gd@gdBddClmZeeeedDe dEefdFdGZee"dDe dEe"fdHdGZdIe fdJdKZedLedMdDe d5e'fdNdOZedPdQdRZedSdTdUZedVedWdXdYZedZd[d\Zed]d^d_Zed`dadbZejdcdddegdfeejɃdgevdDe fdhdiZejˠdjdkkrejdlddgdfeejɃdmedDe fdndoZejdpdddegdfdgehdDe fdqdrZejdsdddegdfdgegdDe fdtduZejdvdddegdfdgegdDe fdwdxZejdydzddgdfeejɃd{d|Zejd}dzddgdfeejɃd~dZejdddgdfeejɃddZejddegdfeejɃdge`fddZejddgdfeejɃddZedeejɃddZejddzddgdfeejɃdVdgeeqfddZejddzddgdfeejɃddZejddzddgdfeejɃddZejddzddgdfeejɃddZejddzddgdfeejɃddZejddzddgdfeejɃddZedeejɃdgeydDe fddZedeejɃdgejdDe fddZedeejɃdgetdDe fddZedeejɃdVdefddZedeejɃdgekdDe fddZedeejɃdgeedDe fddZedeejɃdge|dDe fddZedeejɃdgezdDe fddZedeejɃdge{dDe fddZedeejɃdge}dDe fddZejddzddgdfeejɃdgeidDe fddZejddzddgdfeejɃdgerdDe fddZejddzddgdfeejɃdgesdDe fddĄZedšeejɃdgeadDe fddDŽZejddzddgdfeejɃdgewdDe fddʄZejdddgdfeejɃdgemdDe fdd̈́ZejdddgdfdgeldDe fddЄZejdddgdfeejɃdgexdDe fddӄZejddzddgdfdgendDe fddքZejddzddgdfdgebdDe fddلZejddzddgdfeejɃdgecdDe fdd܄ZedݡeejɃdge_dDe fdd߄ZeddgeodDe fddZeddgeudDe fddZedeejɃdgepdDe fddZedeejɃdgeddDe fddZejdeegddDe?dIe fddZejdeegddDe=dIe fddZejde&eegddDeAdIe fddZejde&eegddDe>dIe fddZejde&eegdejde&eegdddDeGdIe fddZejݐde&eegdejݐde&eegdddDe@dIe fddZejde&dddZejde&dd efd d Zejݐd eegddDeFdIe fd dZejݐdeegddDedIe fddZeddedIe fddZeݐddedIe fddZejȐddddegeegddDeHdIe fddZeejˠddeejˠdddd Zeejˠd!d"dDe7dIe fd#d$Zeejˠd%d&dDe8dIe fd'd(Z eejˠd)d*dIe fd+d,Z eejˠd-d.dDe9dIe fd/d0Z ejݐd1eegddDe.dIe fd2d3Z ejݐd4eegddDe-dIe fd5d6Z ed7d5e'fd8d9Zeݐd:dDe=dIe fd;d<Zeejˠd=d>d?e~dIe fd@dAZdBdCZe jfdDe d5e&fdEdFZdGZdHefdIdJZdefdHedKeegdfdLefdMdNZdOdPZe3e5e4edfdHedQedRedSedLedKeegdff dTdUZdS(Wz The entry point of inference server. (SRT = SGLang Runtime) This file implements HTTP APIs for the inference engine via fastapi. N)asynccontextmanager) HTTPStatus)AnyAsyncGenerator AsyncIteratorCallableDictListOptionalUnion_register_atexitcOsdSN)argskwargsrrV/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/entrypoints/http_server.py)sr)DependsFastAPI HTTPExceptionRequest)RequestValidationError)CORSMiddleware)ORJSONResponseResponseStreamingResponse)FAKE_BOOTSTRAP_HOSTDisaggregationMode)AnthropicCountTokensRequestAnthropicMessagesRequest)AnthropicServing)_launch_subprocessesinit_tokenizer_managerrun_detokenizer_processrun_scheduler_process)OllamaChatRequestOllamaGenerateRequestOllamaShowRequest) OllamaServing) ChatCompletionRequestClassifyRequestCompletionRequestDetokenizeRequestEmbeddingRequest ErrorResponse ModelCard ModelListResponsesRequestScoringRequestTokenizeRequestV1RerankReqInput)OpenAIServingChat)OpenAIServingClassify)OpenAIServingCompletion)OpenAIServingEmbedding)OpenAIServingRerank)OpenAIServingScore)OpenAIServingDetokenizeOpenAIServingTokenize)execute_warmups)envs)FunctionCallParser) AbortReqAttachHiCacheStorageReqInputCheckWeightsReqInputCloseSessionReqInputConfigureLoggingReqContinueGenerationReqInput!DestroyWeightsUpdateGroupReqInputDumperControlReqInputEmbeddingReqInputGenerateReqInputGetWeightsByNameReqInput-InitWeightsSendGroupForRemoteInstanceReqInputInitWeightsUpdateGroupReqInput"LoadLoRAAdapterFromTensorsReqInputLoadLoRAAdapterReqInputOpenSessionReqInputParseFunctionCallReqPauseGenerationReqInputProfileReqInputReleaseMemoryOccupationReqInputResumeMemoryOccupationReqInput#SendWeightsToRemoteInstanceReqInputSeparateReasoningReqInputSetInternalStateReqSlowDownReqInputUnloadLoRAAdapterReqInputUpdateWeightFromDiskReqInput$UpdateWeightsFromDistributedReqInputUpdateWeightsFromIPCReqInputUpdateWeightsFromTensorReqInputUpdateWeightVersionReqInputVertexGenerateReqInput)MultiTokenizerRouterTokenizerWorkerget_main_process_id$monkey_patch_uvicorn_multiprocessingread_from_shared_memorywrite_data_for_multi_tokenizer)TemplateManager) ServerStatusTokenizerManager)enable_func_timer)?parse_remote_instance_transfer_engine_info_from_scheduler_infos)ReasoningParser)PortArgs ServerArgs)process_tracing_inittrace_set_thread_info)add_prometheus_middleware(add_prometheus_track_response_middlewaredelete_directoryget_bool_env_varkill_process_treeset_uvicorn_logging_configs) AuthLevelapp_has_admin_force_endpoints auth_level)get_exception_traceback) __version__SGLANG_HEALTH_CHECK_TIMEOUT!SGLANG_WAIT_WEIGHTS_READY_TIMEOUTxc@s@eZdZUeeeefed<eed<e ed<dZ e e ed<dS) _GlobalStatetokenizer_managertemplate_managerscheduler_infoN$remote_instance_transfer_engine_info) __name__ __module__ __qualname__r rhr`ra__annotations__rfrrr rrrrrs  r _global_state global_statecCs|adSr r)rrrrset_global_statesrreturncCstSr rrrrrget_global_statesrcst}td|\}}}|jdusJddtjddj|_tdt d|jt ||}t }|j ||j|j|jd |d |_tt|||d |S) z Initialization function for multi-process tokenizer mode. It read args information from shm and inits tokenizer manager for current process. multi_tokenizer_args_Nz0API key is not supported in multi-tokenizer modezipc://F)deletez%Start multi-tokenizer worker process z , ipc_name=)r model_path chat_templatecompletion_templatemax_req_input_len)rrr)rbrdapi_keytempfileNamedTemporaryFilenametokenizer_ipc_nameloggerinfoosgetpidrarfinitialize_templatesrrrrrr)main_pid port_args server_argsrrrrrrinit_multi_tokenizers@    r fast_api_appc CsDt|ddr|j}|j}d}ntIdH}t|d}dtjj}|jr,t t t |j rLt |jd|jdkr?d|}n |jd krHd |}t|ttjtj|j_ttjtj|j_ttjtj|j_ttjtj|j_ttj|j_ttjtj|j_ttj|j_ t!tj|j_"t#tj|j_$t%|jj|j_&d}|j'd krd d l(m)}|}n|j'rd dl(m*}|}|+|j'IdHzd dl,m-}|tjtjdd|d|j_.Wnt/yt0}t12d|Ynw|j3durt4|j|j35dtjIdHt16dt7j8t9|d} | :z dVW| ;dS| ;w)Nis_single_tokenizer_modeF TokenizerrzMultiTokenizer-sglangprefillPrefilldecodeDecodedemor)DemoToolServer) MCPToolServer)OpenAIServingResponsesT)enable_prompt_tokens_detailsenable_force_include_usage tool_serverz2Can not initialize OpenAIServingResponses, error: ,z Warmup ended)targetr)<getattrrwarmup_thread_kwargsrdictrr worker_idenable_metricsrpappri enable_tracernotlp_traces_endpointdisaggregation_moderor7rstateopenai_serving_completionr5openai_serving_chatr8openai_serving_embeddingr6openai_serving_classifyr:openai_serving_scorer9openai_serving_rerankr<openai_serving_tokenizer;openai_serving_detokenizer(ollama_servingr anthropic_servingr)sglang.srt.entrypoints.openai.tool_serverrradd_tool_server/sglang.srt.entrypoints.openai.serving_responsesropenai_serving_responses Exceptionryrwarningwarmupsr=splitr threadingThread_wait_and_warmupstartjoin) rrr thread_labelrrrr traceback warmup_threadrrrlifespans               rDISABLE_OPENAPI_DOCz /openapi.json)r openapi_url*T) allow_originsallow_credentials allow_methods allow_headers)routerrequestexccsh|jjdr|jt|jjd|jd}td|i|jdStd|jt |j|jd}t| |jdS)zEnrich HTTP exception with status code and other details. For /v1/responses, emit OpenAI-style nested error envelope: {"error": {"message": "...", "type": "...", "param": null, "code": }} /v1/responsesNmessagetypeparamcodeerrorcontent status_code)objectrrr) urlpath startswithdetailrrphraserr.str model_dump)rr nested_errorrrrrvalidation_exception_handler|s"  rcst|}t|}|r||kr|d|}n|}|jjdr5|tjjdtjjd}t dd|idSt |tjjtjjd}t d| dS) zOverride FastAPI's default 422 validation error with 400. For /v1/responses, emit OpenAI-style nested error envelope; for other endpoints keep legacy format.  rNrrrr)rrr) rerrorsrrrr BAD_REQUESTrvaluerr.r)rrexc_str errors_strrrerrrrrrs,   raw_requestcsJ|jdd}|jdddd}|dkr#tddgd d d gd d S)z;Validate that the request content-type is application/json.z content-type;)maxsplitrzapplication/jsonheaderz:Unsupported Media Type: Only 'application/json' is allowed value_error)locmsgr)rN)headersgetlowerrr)r content_type media_typerrrvalidate_json_requestsr/healthz/health_generatecstjjrtdtddStjjtjkrtddSt j s+j j dkr+tddSddd}d t}tjjrCtj||n&tjjr`t|d g|d d tjjjtjjkr_t_d _n t|d g|d d fd d}t|}t}t|tkrtdIdHtjj |kr|!tjj"#|dtj$tj_tddSt|tks|!t%dt&|}t%dt&tjj }t'dtd|d|tjj"#|dtj(tj_tddS)a9 Check the health of the inference server by sending a special request to generate one token. If the server is running something, this request will be ignored, so it creates zero overhead. If the server is not running anything, this request will be run, so we know whether the server is healthy. z=Health check request received during shutdown. Returning 503.irrr)max_new_tokens temperature HEALTH_CHECK_rF)rid input_idssampling_params log_metricscs(tj2z3dHW}dS6dSr )rrgenerate_request)_grirrrgenszhealth_generate..genNz%H:%M:%SzNHealth check failed. Server couldn't get a response from detokenizer for last z seconds. tic start time: z. last_heartbeat time: ))rrgracefully_exitrrr server_statusrgStartingr>(SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATIONr rrtime is_image_gen"get_image_gen_health_check_request is_generationrIrrrNULLrrbootstrap_hostbootstrap_roomrHasyncio create_taskHEALTH_CHECK_TIMEOUTsleeplast_receive_tstampcancel rid_to_statepopUpstrftime localtimer UnHealthy)rrrrtasktictic_timelast_receive_timerrrhealth_generatesx             r:z/get_model_infoctdtIdHS)zAGet the model information (deprecated - use /model_info instead).zsEndpoint '/get_model_info' is deprecated and will be removed in a future version. Please use '/model_info' instead.N)rr model_inforrrrget_model_info  r= /model_infoc s`tjj}tjjtjjjtjjtjjjtjjj|j |j t |j ddt |j ddtjjjd }|S)zGet the model information. model_typeN architectures) rtokenizer_pathr&preferred_sampling_paramsweight_versionhas_image_understandinghas_audio_understandingr@rArD) rr model_configrrrBr&rCrDis_image_understandable_modelis_audio_understandable_modelr hf_config)rGresultrrrr<)s   r<z/get_weight_versionz/weight_versioncstddd)zGet the current weight version.zdEndpoint '/get_weight_version' or '/weight_version' is deprecated. Please use '/model_info' instead.)rr)rrrrrrD=s rDz/get_server_infocr;)zCGet the server information (deprecated - use /server_info instead).zuEndpoint '/get_server_info' is deprecated and will be removed in a future version. Please use '/server_info' instead.N)rr server_inforrrrget_server_infoGr>rNz /server_infocsJtjIdH}ttjjdrtjj`ittjjtj|t dS)zGet the server information.NrG)internal_statesversion) rrget_internal_statehasattrrrG dataclassesasdictrrz)rOrrrrMQs rMz /get_loadcstdtjIdHS)z6Get load metrics (deprecated - use /v1/loads instead).zkEndpoint '/get_load' is deprecated and will be removed in a future version. Please use '/v1/loads' instead.N)rrrrget_loadrrrrrUes rUz/set_internal_statePOSTPUT)methodsobjcstj|IdH}|Sr )rrset_internal_state)rYrresrrrrZqsrZSGLANG_DUMPER_SERVER_PORTreusez/dumper/{method}methodcs|IdH}|r|IdHni}t||d}tj|IdH}tdd|Dr:dd|D}tdd|idSd d|DS) N)r^bodycss|]}|j VqdSr )success.0rrrr sz*_dumper_control_handler..cSsg|]}|js|jqSr)r`rrarrr sz+_dumper_control_handler..rrrcSsg|] }|jD]}|qqSr)response)rbrKxrrrres)r_jsonrGrrdumper_controlanyr)r^r body_bytesr_rYresultsrrrr_dumper_control_handler{s rm /generatec sjrdttffdd }t|dtjdSztjIdH}|WSt yI}zt d|t |WYd}~Sd}~ww)zHandle a generate request.rc sztj2z3dHW}dtj|tjddVq 6Wn/tyN}z#ddt|ii}t d|dtj|tjddVWYd}~nd}~wwdVdS)Nsdata: )options rr[http_server] Error: sdata: [DONE] ) rrrorjsondumpsOPT_NON_STR_KEYS ValueErrorrrr)outerYrrrstream_resultss, z(generate_request..stream_resultstext/event-stream)r  backgroundNrp) streamrbytesrrrcreate_abort_taskr __anext__rtrr_create_error_response)rYrrxretrvrrwrrs( r/encodec Nztj||IdH}|WSty&}z t|WYd}~Sd}~ww)zHandle an embedding request.Nrrrr~rtrrYrrrvrrrencode_requestrz /classifyc r)zdHandle a reward model request. Now the arguments and return values are the same as embedding models.Nrrrrrclassify_requestrrz /flush_cacheGETc.tjIdH}td|jrddStjdS)zFlush the radix cache.NzCache flushed. Please check backend logs for more details. (When there are running or waiting requests, the operation will not be performed.) rr)rr flush_cacherr`rrrrrrrsrz/clear_hicache_storage_backendcr)z4Deprecated: use POST /hicache/storage-backend/clear.NzjDeprecated endpoint. Use POST /hicache/storage-backend/clear. Hierarchical cache storage backend cleared. rrrrclear_hicache_storagerr`rrrrrr(clear_hicache_storage_backend_deprecatedsrz/hicache/storage-backend/clearcr)z-Clear the hierarchical cache storage backend.Nz,Hierarchical cache storage backend cleared. rrrrrrrclear_hicache_storage_backendsrz/hicache/storage-backendcsvtjjjs tStjj|j|j|j|j dIdH}t |dd}t |j r&dnd|r-|dnd|j r6dd St jd S) zwAttach (enable) HiCache storage backend at runtime. Only allowed when there are NO running / queued requests. )hicache_storage_backend)hicache_storage_backend_extra_config_jsonhicache_storage_prefetch_policyhicache_write_policyNrrz"HiCache storage backend attached. z*Failed to attach HiCache storage backend.  rr)rrr admin_api_key_admin_api_key_missing_responseattach_hicache_storagerrrrrrr`rr)rYrrrrrattach_hicache_storage_backends(   rDELETEcsdtjjjs tStjIdH}t|dd}t|jrdnd|r$|dnd|jr-ddSt j dS) zxDetach (disable) HiCache storage backend at runtime. Only allowed when there are NO running / queued requests. Nrrz"HiCache storage backend detached. z*Failed to detach HiCache storage backend. rrr) rrrrrdetach_hicache_storagerrr`rr)rrrrrdetach_hicache_storage_backends   rcs8tjjjs tStjjjtjjjtjjjtjjjdS)zAGet current HiCache storage backend status (tokenizer-side view).)r$hicache_storage_backend_extra_configrr) rrrrrrrrrrrrrhicache_storage_backend_status1s rz/start_profilec sV|durt}tjj|j|j|j|j|j|j |j |j |j |j d IdHtdddS)zStart profiling.N) output_dir start_step num_steps activities with_stack record_shapesprofile_by_stagemerge_profilesprofile_prefixprofile_stageszStart profiling. rr)rRrr start_profilerrrrrrrrrrr)rYrrrstart_profile_async@s&  rz /stop_profilectjIdHtdddS)zStop profiling.Nz*Stop profiling. This will take some time. rr)rr stop_profilerrrrrstop_profile_asyncY rz /freeze_gccr)z0 See engine.freeze_gc for more details. NzGarbage collection frozen. rr)rr freeze_gcrrrrrfreeze_gc_asyncds rz!/start_expert_distribution_recordcr)zJStart recording the expert distribution. Clear the previous record if any.Nz)Start recording the expert distribution. rr)rr start_expert_distribution_recordrrrrr&start_expert_distribution_record_asyncqrrz /stop_expert_distribution_recordcr)z'Stop recording the expert distribution.Nz(Stop recording the expert distribution. rr)rrstop_expert_distribution_recordrrrrr%stop_expert_distribution_record_async|rrz /dump_expert_distribution_recordcr)z Dump expert distribution record.Nz!Dump expert distribution record. rr)rrdump_expert_distribution_recordrrrrr%dump_expert_distribution_record_asyncrrz/update_weights_from_diskcsHtj||IdH\}}}|||d}|rt|tjdSt|tjdS)zEUpdate the weights from disk inplace without re-launching the server.N)r`rnum_paused_requestsr)rrupdate_weights_from_diskrrOKr)rYrr`rrrrrrrs rz,/init_weights_send_group_for_remote_instancecBtj||IdH\}}||d}|rt|ddSt|tjdSNr`rrr)rr+init_weights_send_group_for_remote_instancerrrrYrr`rrrrrr  rz /send_weights_to_remote_instancecrr)rrsend_weights_to_remote_instancerrrrrrrrrrz)/get_remote_instance_transfer_engine_inforankc s|dus |dkrttjdStjdusttjdkr!ttjdSz |tj|d}|WStyL}ztd|ttjdWYd}~Sd}~ww)Nrr)rrz Exception: ) rrrrrlenrrr)rrKrvrrr(get_remote_instance_transfer_engine_infos$   rz/init_weights_update_groupcBtj||IdH\}}||d}|rt|ddSt|tjdS)z&Initialize the parameter update group.Nrrr)rrinit_weights_update_grouprrrrrrrrs  rz/destroy_weights_update_groupc>tj||IdH\}}||d}t||rddStjdS)z#Destroy the parameter update group.Nrrr)rrdestroy_weights_update_grouprrrrrrrrs rz/update_weights_from_tensorcr)aUpdate the weights from tensor inplace without re-launching the server. Notes: 1. Ensure that the model is on the correct device (e.g., GPU) before calling this endpoint. If the model is moved to the CPU unexpectedly, it may cause performance issues or runtime errors. 2. HTTP will transmit only the metadata of the tensor, while the tensor itself will be directly copied to the model. 3. Any binary data in the named tensors should be base64 encoded. Nrrr)rrupdate_weights_from_tensorrrrrrrrrs  rz /update_weights_from_distributedcr)z/Update model parameter from distributed online.Nrrr)rrupdate_weights_from_distributedrrrrrrrrs  rz/update_weights_from_ipccsRtj||IdH\}}||d}|r"tjjdurdtj_t|St|tjdS)z\Update the weights from IPC (Inter-Process Communication) for checkpoint-engine integration.NrFTr)rrupdate_weights_from_ipcinitial_weights_loadedrrrrrrrr.s  rz/update_weight_versionc s|jr tjjddz|jtjj_tdd|j|jdtj dWSt yC}ztddt |dtj dWYd }~Sd }~ww) zFUpdate the weight version. This operation requires no active requests.T) abort_allzWeight version updated to )r`r new_versionrFz!Failed to update weight version: rN) abort_all_requestsrr abort_requestrrrDrrrrrrrYrrvrrrupdate_weight_version?s*   rz/get_weights_by_namec sdztj||IdH}|durtdWSt|ddWSty1}z t|WYd}~Sd}~ww)zGet model parameter by name.NzGet parameter by name failedrr)rrget_weights_by_namerrrrrrrr^s rz/release_memory_occupationc Jz tj||IdHWdSty$}z t|WYd}~Sd}~ww)z*Release GPU memory occupation temporarily.N)rrrelease_memory_occupationrrrrrrrlrz/resume_memory_occupationc r)zResume GPU memory occupation.N)rrresume_memory_occupationrrrrrrrxrrz/weights_checkercs:tj||IdH\}}t||d|rddStjdSr)rr check_weightsrrr)rYrr`rrrrrsrz /slow_downc r)aBSlow down the system deliberately. Only for testing. Example scenario: when we want to test performance of D in large-scale PD disaggregation and have no enough nodes for P, we can use this to slow down D to let it have enough running sequences, and then disable slowdown to let it run in full batch size. N)rr slow_downrrrrrrrsrz/load_lora_adapterc8tj||IdH}|jrt|tjdSt|tjdSz8Load a new LoRA adapter without re-launching the server.Nr)rrload_lora_adapterr`rrrrrYrrKrrrrrz/load_lora_adapter_from_tensorscr)zELoad a new LoRA adapter from tensors without re-launching the server.Nr)rrload_lora_adapter_from_tensorsr`rrrrrrrrrs rz/unload_lora_adaptercrr)rrunload_lora_adapterr`rrrrrrrrrrrz /open_sessionc sZztj||IdH}|durtd|WSty,}z t|WYd}~Sd}~ww)z1Open a session, and return its unique session id.NzNFailed to open the session. Check if a session with the same id is still open.)rr open_sessionrr)rYr session_idrvrrrrsrz/close_sessionc sPztj||IdHtddWSty'}z t|WYd}~Sd}~ww)zClose the session.Nrr)rr close_sessionrrrrrrrrs rz/configure_loggingcstj|tddS)z&Configure the request logging options.rr)rrconfigure_loggingrrwrrrrs  rz/abort_requestc sPztjj|j|jdtddWSty'}z t|WYd}~Sd}~ww)zAbort a request.)rrrrN)rrrrrrrrrrrrrs rz/parse_function_callcsBt|j|jd}||j\}}|dd|Dd}t|ddS)zD A native API endpoint to parse function calls from a text. )toolstool_call_parsercSsg|]}|qSr)r)rbcallrrrre sz/parse_function_call_request..) normal_textcallsrr)r?rrparse_non_streamtextr)rYrparserrr response_datarrrparse_function_call_requests rz/separate_reasoningcs6t|j|d}||j\}}||d}t|ddS)zB A native API endpoint to separate reasoning from a text. )r@r)reasoning_textrrr)rkreasoning_parserrrr)rYrrrrrrrrseparate_reasoning_requests rz/pause_generationc&tj|IdHtdddddS)zPause generation.NzGeneration paused successfully.okrstatusrr)rrpause_generationrrwrrrr& rz/continue_generationcr)zContinue generation.Nz"Generation continued successfully.rrrr)rrcontinue_generationrrwrrrr1rrz/v1/completions) dependenciesc|jjj||IdHS)z+OpenAI-compatible text completion endpoint.N)rrrhandle_requestrrrrropenai_v1_completions?  r/v1/chat/completionscrz+OpenAI-compatible chat completion endpoint.Nrrrrrrrropenai_v1_chat_completionsG  rz/v1/embeddings)response_classrcr)z&OpenAI-compatible embeddings endpoint.N)rrrrrrrropenai_v1_embeddingsQ  rz /v1/classifycr)z*OpenAI-compatible classification endpoint.N)rrrrrrrropenai_v1_classify]rrz /v1/tokenizez /tokenizeF)rrinclude_in_schemacr)z(OpenAI-compatible tokenization endpoint.N)rrrrrrrropenai_v1_tokenizei  rz/v1/detokenizez /detokenizecr)z*OpenAI-compatible detokenization endpoint.N)rrrrrrrropenai_v1_detokenize{rr z /v1/models)rc stjjg}g}|D]}|t||tjjjdq tjjjr=tjj }| D]\}}|t|j |j |dddq)t|dS)z2Show available models. OpenAI-compatible endpoint.idroot max_model_lenrN)r r parentr )data)rrserved_model_nameappendr/rG context_lenr enable_lora lora_registryget_all_adaptersitems lora_name lora_pathr0)served_model_names model_cardsrrrlora_refrrravailable_modelss.   rz/v1/models/{model:path}modelcsJtjjg}||vrtddd|dddddid St||tjjjd S) zHRetrieves a model instance, providing basic information about the model.rLrz The model 'z' does not existinvalid_request_errorrmodel_not_foundrrr )rrrrr/rGr)rrrrrretrieve_models$   r z /v1/scorecr)zYEndpoint for the decoder-only scoring API. See Engine.score() for detailed documentation.N)rrrrrrrrv1_score_requestrr!rcsJtdi|}|jjj||IdH}t|tr#t|dddddS|S)z6Endpoint for the responses API with reasoning support.Nryzno-cachez keep-alive)z Cache-Control Connection)r r r)r1rrrcreate_responses isinstancerr)rr request_objrKrrrv1_responses_requests  r&z/v1/responses/{response_id} response_idc|jjj|IdHS)zRetrieve a response by ID.N)rrrretrieve_responsesr'rrrrv1_retrieve_responses  r+z"/v1/responses/{response_id}/cancelcr()zCancel a background response.N)rrrcancel_responsesr*rrrv1_cancel_responsesr,r.z /v1/rerank)rXrcr)z:Endpoint for reranking documents based on query relevance.N)rrrrrrrrv1_rerank_requestrr/SGLANG_OLLAMA_ROOT_ROUTE/csdS)z1Ollama-compatible root endpoint for health check.zOllama is runningrrrrr ollama_rootsr2SGLANG_OLLAMA_CHAT_ROUTEz /api/chatcr)z Ollama-compatible chat endpoint.N)rrr handle_chatrrrr ollama_chatsr5SGLANG_OLLAMA_GENERATE_ROUTEz /api/generatecr)z$Ollama-compatible generate endpoint.N)rrrhandle_generaterrrrollama_generate rr8SGLANG_OLLAMA_TAGS_ROUTEz /api/tagscs|jjjS)z'Ollama-compatible list models endpoint.)rrrget_tags)rrrr ollama_tagssr;SGLANG_OLLAMA_SHOW_ROUTEz /api/showcs|jjj|jS)z+Ollama-compatible show model info endpoint.)rrrget_showrrrrr ollama_showsr>z /v1/messagescr)z+Anthropic-compatible Messages API endpoint.N)rrrhandle_messagesrrrranthropic_v1_messages#rr@z/v1/messages/count_tokenscr)z-Anthropic-compatible token counting endpoint.N)rrrhandle_count_tokensrrrranthropic_v1_count_tokens-rrBz/pingcs tddS)z$Check the health of the http server.rr)rrrrrsagemaker_health8s rCz /invocationscrrrrrrrsagemaker_chat_completions>rrDAIP_PREDICT_ROUTEz/vertex_generate vertex_reqcs|jsgSi}dD]|jdr"fdd|jD|<nq dd|jDp,d}tdi|d|i|jp:i}t||IdH}t|trL|Std|iS) N)rr input_embedsrcsg|]}|qSrr rbinstance input_keyrrrePs z#vertex_generate..cSs$g|]}|ddur|dqS) image_dataNrHrIrrrreTs rM predictionsr) instancesr rI parametersrr$rr)rFrinputsrMreqrrrKrvertex_generateIs6    rScCstddt|iitjdS)Nrrr)rrrr)rvrrrrdsrrcCstddi|dS)NrzThis endpoint requires admin API key, but this server was started without one (admin-api-key). Restart with --admin-api-key to enable.r)rrrrrrqs rziVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAbUlEQVRYhe3VsQ2AMAxE0Y/lIgNQULD/OqyCMgCihCKSG4yRuKuiNH6JLsoEbMACOGBcua9HOR7Y6w6swBwMy0qLTpkeI77qdEBpBFAHBBDAGH8WrwJKI4AAegUCfAKgEgpQDvh3CR3oQCuav58qlAw73kKCSgAAAABJRU5ErkJggg==rc s^i}}jrdj|d<d}tdD]6}tdztj|dd|d}|jd ks9Jd |d |jd }Wnt tj j fyMt }Yqw|s`t d |tt|S|}t|dd}|dr{|rxjsxd} nd} nd} |drdnd} dd| di} jrddtjD| d<jdkr| dd| d<n<|rЈjdkr|drtjjddddtidd d!d"gd#g| dd$d%} nd&gj| d <jdkr| d d| d <jr| d dtj| d<d| dd'<t j!} zjdkr0tj"|| | || dkr| nd(d)}|jd ks(J|jt#j$tj_%W|St &d*d$dd d+t'gjfd,dtjDgd-gjd.} tj"|| | || dkrb| nd/d)}|jd krt &d0|jd1|t#j$tj_%W|St &d2(|jt#j)tj_%W|St*yt }t d |ttYdSw)3NzBearer AuthorizationFr~rr?)timeoutr rzres=z , res.text=Tz%Initialization failed. warmup error: rEr&rrnrrr)rrcSsg|]}gdqS)) r)rbrrrrresz*_execute_server_warmup..rnulluser image_urlrzdata:image/png;base64,)rr]rzDescribe the image.)rr)rolerr)rmessages max_tokensr{rzThe capital city of France isriX)rhr rVz%Start of pd disaggregation warmup ...)rr ignore_eoscs$g|]}|dj|jqS)l)dp_sizetp_size)rbirrrres)rXrYrZ )rr(r)riz6End of prefill disaggregation mode warmup with status z, resp: z;Prefill disaggregation mode warm Up Failed, status code: {})+rrranger#r-requestsr rrAssertionError exceptionsRequestExceptionryrrrtrrrhboolskip_tokenizer_initrbrrrrMINIMUM_PNG_PICTURE_BASE64debug_tensor_dump_input_filer1nploadtolistr>SGLANG_WARMUP_TIMEOUTpostrgr2r rrformatr5r) rr rr`rr[last_tracebackr<is_vlm request_namer json_datawarmup_timeoutrrr_execute_server_warmups              0        rzlaunch_callbackexecute_warmup_funccCsl|jrt|js||sdSntjtj_t d|j r"t |j |j r+tt|dur4|dSdS)Nz)The server is fired up and ready to roll!)+checkpoint_engine_wait_weights_before_ready_wait_weights_readyskip_server_warmuprgr2rrr rrdelete_ckpt_after_loadingrrrrnrtrrrr{r|rrrrs     rcCslt}t}t|D]}tjjr!tdt|dddStdq t d|dtjjdS)z:Wait for weights to be ready within the specified timeout.zWeights are ready after z.2fz secondsNrz$Weights are not ready after waiting z} seconds. Consider increasing SGLANG_WAIT_WEIGHTS_READY_TIMEOUT environment variable. Current status: initial_weights_loaded=) WAIT_WEIGHTS_READY_TIMEOUTr#rfrrrrrr-r)rV start_timerrrrr~8s  r~init_tokenizer_manager_funcrun_scheduler_process_funcrun_detokenizer_process_funcc Cst||||d\}}}} t|} tt|||d| d|jr#tt|jdkrPdt_|t_ t |||dt_ |j s@|j s@ttrOddlm} | t|j |j dn d t_t| ||d} zZt||jdkrytjt|j|j|j|jps|jd d d n)dd lm} dgdd d| dd<ttjd|j|j|j|jp|jd d |jdW|jdkr| tjj !dSdS|jdkr| tjj !ww)a3 Launch SRT (SGLang Runtime) Server. The SRT server consists of an HTTP server and an SRT engine. - HTTP server: A FastAPI server that routes requests to the engine. - The engine consists of three components: 1. TokenizerManager: Tokenizes the requests and sends them to the scheduler. 2. Scheduler (subprocess): Receives requests from the Tokenizer Manager, schedules batches, forwards them, and sends the output tokens to the Detokenizer Manager. 3. DetokenizerManager (subprocess): Detokenizes the output tokens and sends the result back to the Tokenizer Manager. Note: 1. The HTTP server, Engine, and TokenizerManager all run in the main process. 2. Inter-process communication is done through IPC (each process uses a different port) via the ZMQ library. )rrrrr)rrrrrTr)add_api_key_middleware)rrFrUuvloop)hostport root_path log_leveltimeout_keep_aliveloop)LOGGING_CONFIGdefaultINFO)handlerslevel propagateloggersz"sglang.srt.entrypoints.http_serverz&sglang.srt.entrypoints.http_server:app)rrrrrrworkersN)"r!rjrrrrqrtokenizer_worker_numrrrrrrrwsglang.srt.utils.authrreruuvicornrunrrfastapi_root_pathlog_level_httpruvicorn.configrrcunlinkrrsocket_mappingclear_all_sockets)rrrrr|r{rrscheduler_infosrrrmulti_tokenizer_args_shmrrrr launch_serverMs           rr (__doc__r*rSloggingrrrr# contextlibrhttprtypingrrrrrr r r setattrnumpyrorqrgrrfastapirrrrfastapi.exceptionsrfastapi.middleware.corsrfastapi.responsesrrrsglang.srt.disaggregation.utilsrr)sglang.srt.entrypoints.anthropic.protocolrr(sglang.srt.entrypoints.anthropic.servingr sglang.srt.entrypoints.enginer!r"r#r$&sglang.srt.entrypoints.ollama.protocolr%r&r'%sglang.srt.entrypoints.ollama.servingr(&sglang.srt.entrypoints.openai.protocolr)r*r+r,r-r.r/r0r1r2r3r4*sglang.srt.entrypoints.openai.serving_chatr5.sglang.srt.entrypoints.openai.serving_classifyr61sglang.srt.entrypoints.openai.serving_completionsr7/sglang.srt.entrypoints.openai.serving_embeddingr8,sglang.srt.entrypoints.openai.serving_rerankr9+sglang.srt.entrypoints.openai.serving_scorer:.sglang.srt.entrypoints.openai.serving_tokenizer;r<sglang.srt.entrypoints.warmupr=sglang.srt.environr>-sglang.srt.function_call.function_call_parserr?sglang.srt.managers.io_structr@rArBrCrDrErFrGrHrIrJrKrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_)sglang.srt.managers.multi_tokenizer_mixinr`rarbrcrdre$sglang.srt.managers.template_managerrf%sglang.srt.managers.tokenizer_managerrgrhsglang.srt.metrics.func_timerri;sglang.srt.model_loader.remote_instance_weight_loader_utilsrj"sglang.srt.parser.reasoning_parserrksglang.srt.server_argsrlrmsglang.srt.tracing.tracernrosglang.srt.utilsrprqrrrsrtrurrvrwrx sglang.utilsrysglang.versionrz getLoggerrrset_event_loop_policyEventLoopPolicyintgetenvr,r dataclassrrrrrrrradd_middlewaresglang.srt.entrypoints.v1_loadsrv1_loads_routerinclude_routerexception_handlerrrr r:r=r<rDrNrMrU api_routeADMIN_OPTIONALrZenvironrrmrrrrrrrrrrrrrrrrsrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rr r!rr&r+r.r/headr2r5r8r;r>r@rBrCrDrSrrrrmrzrr~rrrrrs   (     8          "        3o #M        $