o پiN @sldZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlZddlmZddlmZddlmZddlmZddlmZddlmZmZmZmZmZmZmZddl Z!ddl"Z"ddl#Z#dd l$m%Z%m&Z&dd l'm(Z(dd l)m)Z)dd l*m+Z+e,e-Z.d dZ/e/de0fddZ1dee2e0ee(fde0fddZ3ddZ4de5fddZ6de0fddZ7dmde0de5d e0fd!d"Z8Gd#d$d$Z9  %   dnd&ee0fd'd(Z:d)ee0e;ffd*d+Zfd1d2Z?d3e>fd4d5Z@d6e0fd7d8ZAGd9d:d:ZBdpd;e0dee0fdd?ZEd@e0fdAdBZFeGZHdqdEdFZIdGdHZJdIe0dejKfdJdKZLdrdIe0dMe0dNe>fdOdPZMdQdRZNdSeeddfdTdUZOdVePdWee>deDfdXdYZQ    Zdsd;e0dWee>dSeed[ee2d\e>ddf d]d^ZR  dtd_e0dWe>dSeejKddfd`daZSGdbdcdcZTdddeZUdfdgZVdhdiZWdje0defdkdlZXdS)uzCommon utilitiesN) OrderedDict)ThreadPoolExecutorwraps)BytesIO)dumps)AnyCallableListOptionalTupleTypeUnion)HTMLdisplay) BaseModel)tqdm)envscsdtfdd}|S)Ncss |i|ddSdS)NT)argskwargsfunchas_runr@/home/ubuntu/.local/lib/python3.10/site-packages/sglang/utils.pywrapper&szexecute_once..wrapperr)rrrrr execute_once#srmessagecCst|dSN)loggerinfo)rrrr info_once0sr! json_schemareturncCs^t|tr t|}|St|tr|}|St|tr#t|}|Std|ddd)a+Convert a JSON schema to a string. Parameters ---------- json_schema The JSON schema. Returns ------- str The JSON schema converted to a string. Raises ------ ValueError If the schema is not a dictionary, a string or a Pydantic class. zCannot parse schema z. The schema must be either zBa Pydantic class, a dictionary or a string that contains the JSON zschema specification) isinstancedictjsonrstr issubclassrmodel_json_schema ValueError)r" schema_strrrrconvert_json_schema_to_str5s     r,cCs&t\}}}dt|||}|S)N)sysexc_infojoin tracebackformat_exception)etypevaluetberr_strrrrget_exception_tracebackSsr7valuescs:t|dkrdSt|dtfdd|ddDS)z;Return whether the elements in values are of the same type.Trc3s|]}t|VqdSr)r$).0vtrr _szis_same_type..N)lentypeall)r8rr<r is_same_typeYs  rBfilenameccsRt|}|D]}|drqt|VqWddS1s"wYdS)zRead a JSONL file.#N)open startswithr&loads)rCfinlinerrr read_jsonlbs  "rJwstatesmodecCsddlm}t||=}t|D]/\}}t|trnt||r$|}nt|}|dd|ddd|dddqWddS1sKwYdS) z"Dump program state in a text file.r) ProgramStatez(========================================  zP================================================================================z N)sglang.lang.interpreterrNrE enumerater$r'textwrite)rCrLrMrNfoutisrrrdump_state_textks     &"rXc@s(eZdZddZddZeddZdS) HttpResponsecC ||_dSr)resp)selfr[rrr__init__~ zHttpResponse.__init__cCst|jSr)r&rGr[readr\rrrr&szHttpResponse.jsoncCs|jjSr)r[statusr`rrr status_codeszHttpResponse.status_codeN)__name__ __module__ __qualname__r]r&propertyrbrrrrrY}s rYFmethodc Csddi}|durd||d<|rtj||d|dStjj|||d}|dur*d}ntt|d d }z/tjd krT|rFt |t rFt j |d } nt } tjj ||| d } n tjj |||d} t| WStjjyy} z t| WYd} ~ Sd} ~ ww)zs z'encode_video_base64..zvideo:rk)r VideoCaptureisOpenedIOErrorintgetCAP_PROP_FRAME_COUNTprintnplinspaceranger_appendreleaser?rlistmaprr0rrr) rrrcap total_frames frame_indices_retrexecutorencoded_frames video_bytes video_base64rrrencode_video_base64s0       rcpcCs|dkr|dks@|dkr|dks@|dkr|dks@|dkr |dks@|d kr(|d ks@|d kr0|d ks@|d kr8|dks@|dkrB|dkrBdSdS)z6Checks whether CP is the codepoint of a CJK character.iNii4iMiiߦii?i@ii iiiiiTFr)rrrr_is_chinese_chars rrScCsl|dr|St|dkrtt|dr|St|dkr+tt|dr+|ddS|d|ddS)zPReturns the longest printable substring of text that contains only entire words.rPrrr9NrO)endswithr?rordrfind)rSrrrfind_printable_texts  rc@s@eZdZdZdedefddZddZdefd d Zd d Zd S) LazyImportz/Lazy import to make `import sglang` run faster. module_name class_namecCs||_||_d|_dSr)rr_module)r\rrrrrr]4s zLazyImport.__init__cCs*|jdurt|j}t||j|_|jSr)r importlib import_modulergetattrr)r\modulerrr_load9s  zLazyImport._loadnamecCs|}t||Sr)rr)r\rrrrr __getattr__?s zLazyImport.__getattr__cOs|}||i|Sr)r)r\rrrrrr__call__CszLazyImport.__call__N) rcrdre__doc__r'r]rrrrrrrr1s  rr~c Cs |durtjd|dd}tj|r|Std|d|tj|dd}|t |j d d }d }t |d ?}t ||d dd d}|j |dD]}|||t|qNWdn1sgwYWd|SWd|S1swY|S)z!Read and cache a file from a url.Nz/tmp/rzDownloading from z to Trizcontent-lengthriwbB)desctotalunit unit_scale unit_divisor) chunk_size)ospathr0splitexistsrrrrraise_for_statusrrjrEr iter_contentrTupdater?)r~rCresponse total_sizerfbarchunkrrrdownload_and_cache_fileHs6  (   rcCs tjSr)rSGLANG_IS_IN_CIrrrrris_in_cijr^r html_contentcCs:trt|dd}ttd|ddSt|dS)NrPz
z z )rr'replacerrr)rrrrprint_highlightns r0u@c Cstt||}t||D]/}ttjtj}|tjtj dz| ||f||fWStj y=| Yqwt d)z Reserve an available port by trying to bind a socket. Returns a tuple (port, lock_socket) where `lock_socket` is kept open to hold the lock. r9zNo free port available.)rrrandomshufflesocketAF_INET SOCK_STREAM setsockopt SOL_SOCKET SO_REUSEADDRbindr|close RuntimeError)hoststartend candidatesportsockrrr reserve_portys rc CsDz|WdSty!}ztd|WYd}~dSd}~ww)z? Release the reserved port by closing the lock socket. zError closing socket: N)r Exceptionr) lock_socketrrrr release_ports rcommandcCs|dddd}|}tj}d}|t|krM||}d|vrL|dsL|dsL|d\}}}|rL|durL|d d rL|||<|d 7}q ||d}|sYt d t j |d t j |dS)z Execute a shell command and return its process handle. Supports leading KEY=VALUE env vars (e.g. "VAR=1 python script.py") so that notebook/CI commands work without requiring shell=True. z\ rO\r=-rNrr-r9zECommand contains only environment variable assignments, no executableT)rSstderrenv) rrrenvironcopyr?rF partitionisalnumr* subprocessPopenSTDOUT)rpartsr rVpartkeyrr4rrrexecute_shell_commands&   r0.0.0.0rrcCsH|dur t|\}}nd}|d|}t|}|dur |t|<||fS)zj Launch the server using the given command. If no port is specified, a free port is reserved. Nz --port )rrprocess_socket_map)rrrr full_commandprocessrrrlaunch_server_cmdsrcCs:ddlm}||jt|d}|durt|dSdS)zL Terminate the process and automatically release the reserved port. r)kill_process_treeN)sglang.srt.utilsrpidrpopr)rrrrrrterminate_processs    rrcCs||durdSt|dr|}|durtd|dSt|dr:|srrrr3s         zTypeBasedDispatcher.__call__N)r;r4) rcrdrer r r r r]r:r@rrrrrrr4s  r4cCsNd}tt|t|}t|ddD]}||d|r |}nq||dS)z Finds the largest suffix of 'existing_text' that is a prefix of 'new_chunk' and removes that overlap from the start of 'new_chunk'. rrN)minr?rr) existing_text new_chunk max_overlap max_possiblerVrrr trim_overlapMs rIcCs8d}|j||ddD]}|d}t||}||7}q |S)z] 1) Streams the text, 2) Removes chunk overlaps, 3) Returns the merged text. r-TrrS)generaterI)llmpromptsampling_params final_textr chunk_text cleaned_chunkrrrstream_and_merge[s   rQcCsTd}|j||ddIdH}|2z3dHW}|d}t||}||7}|Vq6dS)z| Streams tokens asynchronously, removes chunk overlaps, and yields the cleaned chunk in real time for printing. r-TrNrS)async_generaterI)rKrLrMrN generatorrrOrPrrrasync_stream_and_mergeis rTqualnamecCs$|dd\}}t|}t||S)z8 Resolve an object by its fully qualified name. .r9)rsplitrrr)rUrobj_namerrrrresolve_obj_by_qualnamews  rY)rK)NFNNN)rr)rr)rN)NNNr*)NN)Yrrr&loggingrrrryrr.r'r1urllib.requestrtweakref collectionsrconcurrent.futuresr functoolsriorrtypingrr r r r r rnumpyrrrrIPython.displayrrpydanticrrsglang.srt.environr getLoggerrcrrr'r!r%r,r7rrBrJrXrYrrwrrrrrrrrboolrrWeakKeyDictionaryrrrrrrrr$floatr)r1r3r4rIrQrTrYrrrrs     $         '."     .