o
    iN                     @   s$  d dl Z d dlmZmZ d dlmZ d dlmZ d dlm	Z
 d dlZd dlmZmZmZmZmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlm Z m!Z! eZ"eegedB f Z#ee"egee f Z$de%d fddZ&dede%d fddZ'dedefddZ(dS )    N)	AwaitableCallable)
HTTPStatus)Any)	APIRouterDependsFastAPIHTTPExceptionRequest)JSONResponseResponse)base)ErrorResponse)OpenAIServing)validate_json_requesthealth)POOLING_TASKSSupportedTasksupported_tasks)r   .c                 C   s  g }d| v r0ddl m}m} ddlm} ddlm}m} ddlm	} ||||ff|||ffg7 }d| v rKddl
m}m}	 dd	lm}
 ||
|	|ffg7 }d
| v rfddlm}m} ddlm} ||||ffg7 }d| v rddlm}m} ddlm} ||||ffg7 }d| v sd| v rddlm}m} ddlm} ||||ffg7 }tdd | D rddlm}m} ddl m!} ||||ffg7 }|S )Ngenerater   )chatcreate_chat_completion)ChatCompletionRequest)
completioncreate_completion)CompletionRequestembed)create_embedding	embedding)EmbeddingRequestclassify)r!   create_classify)ClassificationRequestscore)	do_rerankrerank)RerankRequest)create_scorer$   )ScoreRequestc                 s   s    | ]}|t v V  qd S N)r   ).0task r-   [/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/entrypoints/sagemaker/api_router.py	<genexpr>X   s    z'get_invocation_types.<locals>.<genexpr>)create_poolingpooling)PoolingRequest)"2vllm.entrypoints.openai.chat_completion.api_routerr   r   0vllm.entrypoints.openai.chat_completion.protocolr   -vllm.entrypoints.openai.completion.api_routerr   r   +vllm.entrypoints.openai.completion.protocolr   )vllm.entrypoints.pooling.embed.api_routerr   r   'vllm.entrypoints.pooling.embed.protocolr    ,vllm.entrypoints.pooling.classify.api_routerr!   r"   *vllm.entrypoints.pooling.classify.protocolr#   )vllm.entrypoints.pooling.score.api_routerr%   r&   'vllm.entrypoints.pooling.score.protocolr'   r(   r$   r)   any+vllm.entrypoints.pooling.pooling.api_routerr0   r1   )vllm.entrypoints.pooling.pooling.protocolr2   )r   INVOCATION_TYPESr   r   r   r   r   r   r   r   r    r!   r"   r#   r%   r&   r'   r(   r$   r)   r0   r1   r2   r-   r-   r.   get_invocation_types   sR   






rA   appc              
      s   t  }t|}dd |D  |jdtd|jdtdtjdtdtfdd}|jd	tt	gt
jjd
tit
jjd
tit
jjd
tiidtjt tjd
ddtf fdd}| | d S )Nc                 S   s&   g | ]\}\}}t |||ffqS r-   )pydanticTypeAdapter)r+   request_typeget_handlerendpointr-   r-   r.   
<listcomp>h   s    
z!attach_router.<locals>.<listcomp>z/ping)response_classraw_requestreturnc                    s   t | I dH S )z+Ping check. Endpoint required for SageMakerNr   rJ   r-   r-   r.   pingm   s   zattach_router.<locals>.pingz/invocationsmodel)dependencies	responses)adapter_pathc           
   
      s   z	   I dH }W n t jy# } zttjjd| d|d}~ww  fddD }|D ]\}}z||}W n
 tjyD   Y q/w || I dH   S fdd|D }d| }t	 j
|d}	t|	 |	jjd	S )
z9For SageMaker, routes requests based on the request type.NzJSON decode error: )status_codedetailc                    s(   g | ]\}\}}| d ur||fqS r*   r-   )r+   	validatorrF   rG   rL   r-   r.   rH      s
    
z6attach_router.<locals>.invocations.<locals>.<listcomp>c                    s.   g | ]\}}t |j  tr jnt qS r-   )
isinstance_typetype__name__str)r+   rT   _)tr-   r.   rH      s    z;Cannot find suitable handler for request. Expected one of: )message)contentrR   )jsonJSONDecodeErrorr	   r   BAD_REQUESTvaluevalidate_pythonrC   ValidationErrorr   create_error_responser   
model_dumperrorcode)
rJ   bodyevalid_endpointsrequest_validatorrG   request
type_namesmsgresINVOCATION_VALIDATORS)rJ   r[   r.   invocationst   s8   


z"attach_router.<locals>.invocations)r   rA   postr   getsagemaker_standardsregister_ping_handlerr
   r   r   r   r`   ra   r   UNSUPPORTED_MEDIA_TYPEINTERNAL_SERVER_ERRORregister_invocation_handlerstateful_session_managerinject_adapter_idinclude_router)rB   r   routerr@   rM   rr   r-   rp   r.   attach_routerc   s,   	
 r~   rK   c                 C   s
   t | S r*   )ru   	bootstrap)rB   r-   r-   r.   sagemaker_standards_bootstrap   s   
r   ))r^   collections.abcr   r   httpr   typingr   +model_hosting_container_standards.sagemaker	sagemakerru   rC   fastapir   r   r   r	   r
   fastapi.responsesr   r   (vllm.entrypoints.openai.basic.api_routerr   'vllm.entrypoints.openai.engine.protocolr   &vllm.entrypoints.openai.engine.servingr   vllm.entrypoints.openai.utilsr   ,vllm.entrypoints.serve.instrumentator.healthr   
vllm.tasksr   r   RequestTypeGetHandlerFn
EndpointFntuplerA   r~   r   r-   r-   r-   r.   <module>   s(   H@