o i=d@s~UddlZddlmZmZmZddlmZddlmZm Z m Z m Z m Z ddl Z ddlmZddlmZddlmZddlmZdd lmZdd lmZmZdd lmZmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&ddl'm(Z(m)Z)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;mee$Z?e e@d<GdddeZAdS)N)AsyncGeneratorCallableMapping)partial)AnyFinalLiteral TypeAliascast)Request) assert_never) EngineClient)ChatTemplateContentFormatOption) RequestLogger) ErrorResponse UsageInfo) OpenAIServing ServeContext)OpenAIServingModels)EmbeddingBytesResponseEmbeddingChatRequestEmbeddingCompletionRequestEmbeddingRequestEmbeddingResponseEmbeddingResponseData)encode_pooling_bytesencode_pooling_output_base64encode_pooling_output_float) TokensPrompt) init_logger) PoolingOutputPoolingRequestOutput) PoolingParams) TokPrompt)merge_async_iterators) chunk_list) EmbedDType EndiannessEmbeddingServeContextcseZdZdZddddedededBdedBd ed e d e d dffd dZ de d e dBfddZ deedededededdeded efddZdeedededededdeded efddZde d eeBe Bfdd Zd efd!d"Zd e fd#d$Zde d%eed&ed'eeefdBd(ed eeedff d)d*Zd+ed,eed-ed e ffd.d/ Z!de d0e"d&ed'eeefdBd1ed eedff d2d3Z#de d e dBffd4d5 Z$de d e dBffd6d7 Z% d;d+e&d8e'dBd ee Bfd9d:Z(Z)S)<OpenAIServingEmbeddingembdF)trust_request_chat_templatelog_error_stack engine_clientmodelsrequest_loggerN chat_templatechat_template_content_formatr+r,returnc s^tj||||d||_||_||_|jj}t|o|j|_ |r*|j r*|j |_ dSd|_ dS)N)r-r.r/r,) super__init__r0r1r+ model_config pooler_configboolenable_chunked_processingsupports_chunked_processing max_embed_len) selfr-r.r/r0r1r+r,r6 __class__\/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/entrypoints/pooling/embed/serving.pyr41s( zOpenAIServingEmbedding.__init__ctxc szW||j|_t|jtr;|j|jj|jj|jd}|dur#|WS|j |j|jj |j|j ddIdH\}|_ WdSt|jt rS|j|j|jjddIdH|_ WdS|dWSttfyx}ztd|t|WYd}~Sd}~ww)N)request_chat_templatechat_template_kwargsr+)default_templatedefault_template_content_formatdefault_template_kwargs) prompt_input prompt_embedsz#Invalid classification request typez$Error in preprocessing prompt inputs)_maybe_get_adaptersrequest lora_request isinstancer_validate_chat_templater0rBr+_preprocess_chatmessagesr1engine_promptsr_preprocess_completioninputcreate_error_response ValueError TypeErrorlogger exceptionstr)r;r@error_check_ret_er>r>r? _preprocessSs@    z"OpenAIServingEmbedding._preprocessfinal_res_batch request_id created_time model_nameencoding_format)floatbase64 embed_dtype endiannesscCstttgtttBf|dkrtntt||d}g} d} t |D]\} } t | || d} | j }| | | t |7} q!t| | d}t|||| |dS)Nra)rcrdr)index embedding) prompt_tokens total_tokensidcreatedmodeldatausage)r rr!listrarWrrr enumeraterprompt_token_idsappendlenrr)r;r\r]r^r_r`rcrd encode_fnitemsnum_prompt_tokensidx final_resitemrqrnr>r>r?%request_output_to_embed_json_responsexs>   zr>r?&request_output_to_embed_bytes_responses$   z=OpenAIServingEmbedding.request_output_to_embed_bytes_responsec Cs|jj}|jj}|jj}|dks|dkr#||j|j|j|j|||S|dks+|dkr:| |j|j|j|j|||St |dS)Nrarbr{r|) rIr`rcrdrzr\r]r^r_rr )r;r@r`rcrdr>r>r?_build_responses0  z&OpenAIServingEmbedding._build_responsecCs|jjS)z?Get the model's effective maximum sequence length for chunking.)r5 max_model_len)r;r>r>r?_get_max_position_embeddingssz3OpenAIServingEmbedding._get_max_position_embeddingscCst|ttfo |jS)zr>r?_should_use_chunked_processingsz5OpenAIServingEmbedding._should_use_chunked_processing token_idspooling_params trace_headers prompt_idxc sg}|}tt||D]>\}} |jd|d|} t| d} |j| | ||jd|j|j } | } |j j | || |j| ||jj d}||q|S)z1Process a single prompt using chunked processing.z-prompt--chunk-)rqparamsrJrJtokenization_kwargsrpriority)rrpr%r]r _log_inputsrJrIbuild_tok_paramsr5get_encode_kwargsr-encoderrr)r;r@rrrr generatorsmax_pos_embeddings chunk_idx chunk_tokenschunk_request_idchunk_engine_prompt tok_paramsroriginal_generatorr>r>r?_process_chunked_requests6    z/OpenAIServingEmbedding._process_chunked_requestrI input_ids input_textc st|}t|ttfrT||}|}|jdurd}|j}nd}|j}d} d} ||kr6t| j |||d||krN|rDt d||n t| j d||dt ||d St |||S) z>Override to support chunked processing for embedding requests.Nzmaximum embedding input lengthzmaximum context lengthzThis model's {length_type} is {max_length_value} tokens. However, you requested {token_num} tokens in the input for embedding generation. Please reduce the length of the input.zThis model's {length_type} is {max_length_value} tokens. However, you requested {token_num} tokens in the input for embedding generation. Please reduce the length of the input or enable chunked processing.) length_typemax_length_value token_numzOInput length %s exceeds max_position_embeddings %s, will use chunked processingz"maximum position embeddings length)promptrq)rsrKrrrrr:rrSformatrUinforr3_validate_input) r;rIrrrenable_chunkedrrrvalidation_error_msgchunked_processing_error_msgr<r>r?r'sJ    z&OpenAIServingEmbedding._validate_input engine_prompt prompt_indexc s\|jd|}|j||||jd|j|j}|}|jj||||j|||jj dS)zACreate a generator for a single prompt using standard processing.-rr) r]rrJrIrr5rr-rr) r;r@rrrrrequest_id_itemrrr>r>r?_create_single_prompt_generatorrs& z6OpenAIServingEmbedding._create_single_prompt_generatorc s8||j}|st|IdHSg}zp|jdurdn ||jjIdH}||}t|t r3|WS|j dur>| dWS| }t |j D]5\}}d|vrk|d} t| |krk||| |||IdH} || qG||||||IdH} || qGt||_WdSty} z | | WYd} ~ Sd} ~ ww)z'Override to support chunked processing.NEngine prompts not availablerq)rrIr3_prepare_generators raw_request_get_trace_headersr_create_pooling_paramsrKrrOrRrrprsrextendrrrr$result_generator Exception) r;r@ use_chunkedrrrrirrqchunk_generators generatorrZr<r>r?rsN           z*OpenAIServingEmbedding._prepare_generatorsc sz|jdur|dWS||j}|s!tj|dIdHWS|jdur,|dWSi}i}|j2z3dHW\}}d|jvr|jd}z t || dd}Wn t t fy`|}Ynw||vrtdd d |jdd d ||<||} t |ts|d t|jWSt|jd r|jj} nt|jd r|jj} n|dt|jjWSt | tjstj| tjd} |jdur|dWSt|j} | jtjd| } | ddur| | d<n| d| 7<| d| 7<| dd7<q3|jd}zt |d}Wnt t fy|}Ynw|||<q36g} t|j}t|D]}||vr||} | d}| d}|durt |tjrt |t tfr|d kr||}t|d}|j|}d|vrv|d|dWS|d}t| d||d dd}| |q+|d|WS||vr| ||q+|d|WS| |_ WdSt!y}z ||WYd}~Sd}~ww)zCollect and aggregate batch results with support for chunked processing. For chunked requests, performs online aggregation to minimize memory usage. For regular requests, collects results normally. Nr)r@zResult generator not availablerrrr) weighted_sum total_weight chunk_countr]z9Expected PoolingRequestOutput for chunked embedding, got rmrfzUnsupported output type: )dtypez6prompt_token_ids cannot be None for chunked processingrrr)rmrqzChunked prompt z does not contain token IDsr]T)r]rqoutputsnum_cached_tokensfinishedz&Failed to aggregate chunks for prompt zResult not found for prompt )"rOrRrrIr3_collect_batchrr]splitintrerS IndexErrorrKr!type__name__hasattrrrmrftorchTensortensorfloat32rqrstorangerar rrr\r)r;r@rprompt_aggregatorsshort_prompts_results result_idxresultpartsr aggregatorembedding_dataweightweighted_embeddingr\ num_promptsrrfinal_embeddingpooling_output_dataoriginal_promptoriginal_token_idspooling_request_outputrZr<r>r?rs                    O              z%OpenAIServingEmbedding._collect_batchrcsF|j}|jd|||j}t||||d}||IdHS)z Embedding API similar to OpenAI's API. See https://platform.openai.com/docs/api-reference/embeddings/create for the API specification. This API mimics the OpenAI Embedding API. r)rIrr_r]N)r.r_request_id_prefix_base_request_idr]r(handle)r;rIrr_r]r@r>r>r?create_embeddingys  z'OpenAIServingEmbedding.create_embedding)N)*r __module__ __qualname__rr rrrWrr7r4r(rr[ror!rrr&r'rrzrrrrrr"rrrobjectrrr#rrrrr r __classcell__r>r>r<r?r).s   " % 1 "    1K   =.r))Brcollections.abcrrr functoolsrtypingrrrr r rfastapir typing_extensionsr vllm.engine.protocolr vllm.entrypoints.chat_utilsrvllm.entrypoints.loggerr'vllm.entrypoints.openai.engine.protocolrr&vllm.entrypoints.openai.engine.servingrr&vllm.entrypoints.openai.models.servingr'vllm.entrypoints.pooling.embed.protocolrrrrrrvllm.entrypoints.pooling.utilsrrrvllm.inputs.datar vllm.loggerr vllm.outputsr r!vllm.pooling_paramsr"vllm.renderers.inputsr#vllm.utils.async_utilsr$vllm.utils.collection_utilsr%vllm.utils.serial_utilsr&r'rrUr(__annotations__r)r>r>r>r?s4