o پiG@sddlZddlZddlZddlZddlZddlmZmZmZm Z ddl Z ddl Z ddl m Z ddlmZddlmZmZddlmZmZddlmZddlmZmZmZmZmZdd lmZGd d d eZ d d Z!GdddZ"dS)N)DictListOptionalUnion) global_config) BaseBackend)get_chat_templateget_chat_template_by_model_path)ChoicesDecisionChoicesSamplingMethod)StreamExecutor) REGEX_BOOL REGEX_FLOAT REGEX_INT REGEX_STRSglSamplingParams) http_requestc s*eZdZ   d3dedeedeedeeffdd Zdd Zd d Zd d ZddZ defddZ ddZ ddZ de fddZde fddZdefddZde defdd Zde defd!d"Zde d#eed$ed%ed&ef d'd(Zd)eed*efd+d,Zde fd-d.Zde fd/d0Zd1d2ZZS)4RuntimeEndpointNbase_urlapi_keyverifychat_template_namecsttd|_||_||_||_t|jd|j|jd}||||_ |r0t ||_ dSt |j d|_ dS)NTz/get_model_inforr model_path) super__init__support_concate_and_appendrrrr_assert_successjson model_infor chat_templater )selfrrrrres __class__X/home/ubuntu/.local/lib/python3.10/site-packages/sglang/lang/backend/runtime_endpoint.pyrs"    zRuntimeEndpoint.__init__cCs |jdS)Nr)rr!r%r%r&get_model_name8s zRuntimeEndpoint.get_model_namecCs(t|jd|j|jdd}||dS)Nz /flush_cachePOST)rrmethodrrrrrr!r"r%r%r& flush_cache;szRuntimeEndpoint.flush_cachecCs*t|jd|j|jd}|||S)N/get_server_infor)rrrrrrr,r%r%r&get_server_infoDs zRuntimeEndpoint.get_server_infocCs|jSN)r r'r%r%r&rMsz!RuntimeEndpoint.get_chat_template prefix_strcCs2t|jd|ddid|j|jd}||dS)N /generatemax_new_tokensrtextsampling_paramsrrrr+)r!r1r"r%r%r& cache_prefixPs zRuntimeEndpoint.cache_prefixcC&t|jd|j|jd}||dS)Nz/start_profilerr+r,r%r%r& start_profileY zRuntimeEndpoint.start_profilecCr9)Nz /stop_profilerr+r,r%r%r& stop_profilear;zRuntimeEndpoint.stop_profilescCD|jddid}|||t|jd||j|jd}||dSNr3rr4r2r7text_ _add_imagesrrrrrr!r=datar"r%r%r&commit_lazy_operationsi z&RuntimeEndpoint.commit_lazy_operationscCr>r?r@rCr%r%r& fill_imagetrFzRuntimeEndpoint.fill_imager6cCs|jdurdS|jdkrg|_d}|jdtfvr#t}|jddgn.|jdtfvr5t}|jddgn|jdtfvr?t}n|jdt fvrIt }nt d|j|durg|j durgt d |jd |j ||_ dS) Nr%int  floatstrboolzInvalid dtype: z>Both dtype and regex are set. Only dtype will be used. dtype: z , regex: )dtypestoprHrextendrKrrLrrMr RuntimeErrorregexwarningswarn)r!r6 dtype_regexr%r%r&_handle_dtype_to_regexs*   z&RuntimeEndpoint._handle_dtype_to_regexc Cs|||jtjtjd|d}dD]}t||d}|dur&|||<q|||t|j d||j |j d}| || }|d}||dfS)Nskip_special_tokensspaces_between_special_tokensr4return_logproblogprob_start_lentop_logprobs_numreturn_text_in_logprobsr2r7r5 meta_info)rVrArskip_special_tokens_in_output$spaces_between_special_tokens_in_out to_srt_kwargsgetattrrBrrrrrr) r!r=r6rDitemvaluer"objcompr%r%r&generates0     zRuntimeEndpoint.generatec cs|||jtjtjd|d}dD]}t||d}|dur'|||<qd|d<|||t|j d|d|j |j d}| |d}|j d d D]8}|d }|r|d r|d krbdSt|ddd}|d|d} |d} |t| 7}| | fVqMdS)NrWr4rZTstreamr2)rrirrrF)decode_unicodeutf-8data:z data: [DONE]rJr5r_)rVrArr`rarbrcrBrrrrr iter_linesdecode startswithrloadsstriplen) r!r=r6rDrdrer"poschunk chunk_textr_r%r%r&generate_streamsJ       zRuntimeEndpoint.generate_streamchoices temperaturechoices_methodreturncsp|dksJjddid}||}|dd}t|dd}fdd |Dddd d d |d }||}d d |D} dd |D} dd |D} tt| D]8} | | dd} j| r| | dd}| | t| | |t| | d| | <| | dd| | <qT|jrdd | D}|ddid d}||}dd |D}nd}||| | | |dS)Ngh㈵>r3rr4r_ prompt_tokenscsg|]}j|qSr%)rA).0cr=r%r& sz*RuntimeEndpoint.select..)r3ryT)r5r6r[r^r\cSg|]}|ddqSr_input_token_logprobsr%r~rr%r%r&rcSr)r_output_token_logprobsr%rr%r%r&rrcSsg|] }t|ddqSr)"compute_normalized_prompt_logprobsrr%r%r&rscSsg|] }dd|DqS)cSsg|]}|dqS)rr%)r~elr%r%r&r(sz5RuntimeEndpoint.select...r%)r~sublr%r%r&r(s) input_idsr6r[cSrrr%rr%r%r&r/s )rxnormalized_prompt_logprobsrrunconditional_token_logprobs)rA_generate_http_requestmaxrangersendswithrequires_unconditional_logprobs)r!r=rxryrzrDrf prompt_lenr\rrrihealed_token_strhealed_token_logprobrrr%rr&selects`      zRuntimeEndpoint.selectsrc_ridsdst_ridcCs.t|jd||d|j|jd}||dS)Nz/concate_and_append_request)rrr7r+)r!rrr"r%r%r&concatenate_and_append=sz&RuntimeEndpoint.concatenate_and_appendcCs8|||t|jd||j|jd}|||S)Nr2r7)rBrrrrrrrCr%r%r&rFs  z&RuntimeEndpoint._generate_http_requestcCs6|jrt|jdksJd|jdd|d<dSdS)NrzOnly support one image.r image_data)images_rs)r!r=rDr%r%r&rBQszRuntimeEndpoint._add_imagescCsB|jdkrz |}Wt|tjy|j}Yt|wdS)N) status_coderJSONDecodeErrorr5rQ)r!r"contentr%r%r&rVs  zRuntimeEndpoint._assert_success)NNN)__name__ __module__ __qualname__rLrrr(r-r/rr8r:r<r rErGrrVrhrwrrKr r rrrrBr __classcell__r%r%r#r&rs`       ' 2 E  rcCsdd|D}t|t|S)NcSsg|] }|dr|dqS)rr%)r~xr%r%r&r`sz6compute_normalized_prompt_logprobs..)sumrs)input_logprobsvaluesr%r%r&r_src@s(eZdZdZ  d%dedefddZdd Zd d Zd d Z defddZ ddZ d&dede e fddZeZ     d'deeeefde e de eeeefde eeeefde eeeefde ee ef ddZdeeeeee eee ffdd Zd!d"Zd#d$ZdS)(Runtimea5 A wrapper for the HTTP server. This is used for launching the server in a python program without using the command line interface. It is mainly used for the frontend language. You should use the Engine class if you want to do normal offline processing without the frontend language. errorr@ log_levellaunch_timeoutc Osddlm}ddlm}ddlm}||d|i||_t|jjdD]}||r+nq#||j_|j |_ |j d|_ d|_ t d } | j||jfd } | | j |_ t|jt} tN} t| |krz| |j d } | jd kr}Wn-Wn tjyYnw| s|td tdt| |ksl|tdWdn1swYt|j |_dS)aLSee the arguments in server_args.py::ServerArgs Args: log_level: Log level for the server. timeout: Timeout in seconds for waiting for the server to start. *args: Additional arguments passed to ServerArgs. **kwargs: Additional keyword arguments passed to ServerArgs. r) launch_server) ServerArgs)is_port_availableri@r2Nspawn)targetargsz/health_generaterz;Initialization failed. Please see the error messages above.r}z1Server failed to start within the timeout period.)"sglang.srt.entrypoints.http_serverrsglang.srt.server_argsrsglang.srt.utilsr server_argsrporturl generate_urlpidmultiprocessing get_contextProcessstartatexitregistershutdowntimerequestsSessiongetrRequestExceptionis_aliverQsleep TimeoutErrorrendpoint)r!rrrkwargsrrrrctxproc start_timesessionresponser%r%r&rnsV          zRuntime.__init__cCs.ddlm}|jdur||jd|_dSdS)Nr)kill_process_tree)rrr)r!rr%r%r&rs    zRuntime.shutdowncC|jdSr0)rr:r'r%r%r&r:zRuntime.start_profilecCrr0)rr<r'r%r%r&r<rzRuntime.stop_profileprefixcCs|j|dSr0)rr8)r!rr%r%r&r8szRuntime.cache_prefixcCs,ddlm}||jj|jj|jj|jjdS)Nr) get_tokenizer)tokenizer_modetrust_remote_coderevision)&sglang.srt.utils.hf_transformers_utilsrrtokenizer_pathrrr)r!rr%r%r&rs zRuntime.get_tokenizerNpromptr6c CsP|jjr ||dd}n||dd}d}tjdd}tj|dd4IdHv}|j|j|d4IdHS}|j2zA3dHW\}} | d }|ry| d ry|d krRn)t |d d d } d| vrv| d|d} | ro| V|t| 7}q8| Vq86WdIdHn 1IdHswYWdIdHdS1IdHswYdS)NT)rr6ri)r5r6riri0*)total)timeout trust_envrrkrlzdata: [DONE] rmrJr5)rskip_tokenizer_initaiohttp ClientTimeout ClientSessionpostrr iter_chunksrorprrqrrrs) r!rr6 json_datartrrrru_rDcurr%r%r&async_generates<  *.zRuntime.async_generateFr[r\r] lora_pathc CsR||||||d}t|trt|t|ksJtj|jd|d}t|S)N)r5r6r[r\r]rr2r) isinstancelistrsrrrrdumps) r!rr6r[r\r]rrrr%r%r&rhs zRuntime.generatecCs*d|i}tj|jd|d}t|S)Nr5z/encoder)rrrrr)r!rrrr%r%r&encode szRuntime.encodec st4IdHW}||jd4IdH3}|jdkr8|IdHWdIdHWdIdHS|IdH}td|dd1IdHsPwYWdIdHdS1IdHsfwYdS)Nr.rzFailed to get server info. rmessage)rrrrstatusrrQ)r!rr error_datar%r%r&r/s  .zRuntime.get_server_infocCs |dSr0)rr'r%r%r&__del__s zRuntime.__del__)rrr0)NFNNN)rrr__doc__rLrKrrr:r<r8rrrr add_requestrrrMrHrhrr/rr%r%r%r&rdsX  F  $   r)#rrrrrStypingrrrrrrsglang.global_configr sglang.lang.backend.base_backendrsglang.lang.chat_templaterr sglang.lang.choicesr r sglang.lang.interpreterr sglang.lang.irr rrrr sglang.utilsrrrrr%r%r%r&s(    G