o
    `۷i#                     @   s  d dl Z d dlmZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZmZ d dlmZmZmZmZmZmZ d dlmZmZmZmZmZmZmZ d dlmZmZ d d	lm Z  d d
l!m"Z" d dl#m$Z$m%Z% d dl&m'Z'm(Z(m)Z) d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0 d dl1m2Z2 de	defddZ3eege2f Z4dede2fddZ5		d7dedee6 dee4 defddZ7dd Z8dd Z9dd  Z:d!d" Z;d#d$ Z<dee6ee6 f fd%d&Z=ee6ee%ge(f Z>	d8d'e6d(ed)e%d*ee de(f
d+d,Z?d-e"d.e6fd/d0Z@d1ed2efd3d4ZAd5d6 ZBdS )9    N)CallableOptionalTuple)HEAD_NODE_RESOURCE_NAME)	GcsClient)ClusterNodeInfoCacheDefaultClusterNodeInfoCache)CreatePlacementGroupRequestDeploymentHandleSourceDeploymentIDEndpointInfoRequestMetadataRequestProtocol)CONTROLLER_MAX_CONCURRENCYRAY_SERVE_ENABLE_TASK_EVENTS)RAY_SERVE_PROXY_PREFER_LOCAL_NODE_ROUTINGRAY_SERVE_PROXY_USE_GRPC%RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOPSERVE_CONTROLLER_NAMESERVE_NAMESPACE)DefaultDeploymentSchedulerDeploymentScheduler)EventLoopMonitor)gRPCGenericServer)DynamicHandleOptionsInitHandleOptions)CurrentLoopRouterRouterSingletonThreadRouter)asyncio_grpc_exception_handlergenerate_request_idget_current_actor_idget_head_node_idinside_ray_client_contextresolve_deployment_response)PlacementGroup
gcs_clientreturnc                 C   s   t | S N)r   )r&    r)   U/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/serve/_private/default_impl.pycreate_cluster_node_info_cache5   s   r+   requestc                 C   s"   t jj| j| j| j| jd| jdS )Ndetached)_soft_target_node_idnamelifetimebundle_label_selector)rayutilplacement_groupbundlesstrategytarget_node_idr/   r1   )r,   r)   r)   r*   _default_create_placement_group<   s   r8   cluster_node_info_cachehead_node_id_override"create_placement_group_fn_overridec                 C   s   |pt  }t| ||ptdS )N)create_placement_group_fn)r"   r   r8   )r9   r:   r;   head_node_idr)   r)   r*   create_deployment_schedulerI   s   
r>   c                  K      ddl m} |di | S )Nr   )Replicar)   )ray.serve._private.replicar@   )kwargsr@   r)   r)   r*   create_replica_implW      rC   c                  K   r?   )Nr   )ReplicaMetricsManagerr)   )rA   rE   )rB   rE   r)   r)   r*   create_replica_metrics_manager]   rD   rF   c                  K   s   t di | S Nr)   )r   rB   r)   r)   r*   create_dynamic_handle_optionsc   s   rI   c                  K   s   t jdi | S rG   )r   createrH   r)   r)   r*   create_init_handle_optionsg   s   rK   c                 C   s   t jj }tj}| r| jtjkr|j	rtj
}n|jrtj}t|jr%|jnt |jr-|jnt |j|j|j|j|j||j|j| j|j|jdS )N)
request_idinternal_request_idcall_methodrouteapp_namemultiplexed_model_idis_streaming_request_protocolgrpc_context_by_reference_on_separate_looprequest_serializationresponse_serialization)r2   servecontext_get_serve_request_contextr   	UNDEFINED_sourcer
   PROXYis_http_requestHTTPrT   GRPCr   rL   r    _internal_request_idmethod_namerO   rP   rQ   streamrU   _run_router_in_separate_looprW   rX   )init_optionshandle_options_request_contextrequest_protocolr)   r)   r*   get_request_metadatak   s4   rj   c                  C   sZ   t   } zttt  jd}|  || }W | |fS  ty,   d }Y | |fS w )N)address)	r2   get_runtime_contextget_node_idr+   r   gcs_addressupdateget_node_az	Exception)node_idr9   azr)   r)   r*   _get_node_id_and_az   s   rt   	handle_iddeployment_idrg   request_router_classc                 C   s   ddl m} t }t \}}| j}t }	|jr:t}
|jt	j
kr$tj}n|jt	jkr.tj}ntj}t|t nzt  W n tyK   tdw t}
|
||| ||j||	 t|||jdS )Nr   _get_global_clientzNo event loop running. You cannot use a handle initialized with `_run_router_in_separate_loop=False` when not inside an asyncio event loop.)controller_handlerv   ru   self_actor_idhandle_sourcerw   "enable_strict_max_ongoing_requestsresolve_request_arg_funcrr   availability_zoneprefer_local_node_routing)ray.serve.contextry   r!   rt   _controllerr#   re   r   r]   r
   REPLICAr   COMPONENT_REPLICAr^   COMPONENT_PROXYCOMPONENT_UNKNOWN_get_singleton_asyncio_loopset_exception_handlerr   asyncioget_running_loopRuntimeErrorr   r$   _prefer_local_routing)ru   rv   rg   rw   ry   actor_idrr   r   rz   is_inside_ray_client_contextrouter_wrapper_cls	componentr)   r)   r*   create_router   sJ   

r   grpc_serverserver_addressc                 C   s   |  | dS )z3Helper function to add an address to a gRPC server.N)add_insecure_port)r   r   r)   r)   r*   add_grpc_address   s   r   endpointinfoc                 C   sR   ddl m} | }|j| j| jdd}|js|jttj	t
d |j|j t dS )Nr   rx   T)check_exists)r   r]   re   )rd   rU   )r   ry   
get_handler/   rP   is_initialized_initr   r
   r^   r   optionsapp_is_cross_languager   )r   r   ry   clienthandler)   r)   r*   get_proxy_handle   s   r   c                  C   s4   ddl m}  tjttddddtdittd	| }|S )Nr   )ServeControllerr-   gMbP?)	r/   	namespacenum_cpusr0   max_restartsmax_task_retries	resourcesmax_concurrencyenable_task_events)	ray.serve._private.controllerr   r2   remoter   r   r   r   r   )r   controller_implr)   r)   r*   get_controller_impl   s   
r   )NNr(   )Cr   typingr   r   r   r2   ray._common.constantsr   ray._rayletr   *ray.serve._private.cluster_node_info_cacher   r   ray.serve._private.commonr	   r
   r   r   r   r   ray.serve._private.constantsr   r   r   r   r   r   r   'ray.serve._private.deployment_schedulerr   r   (ray.serve._private.event_loop_monitoringr   ray.serve._private.grpc_utilr   !ray.serve._private.handle_optionsr   r   ray.serve._private.routerr   r   r   ray.serve._private.utilsr   r    r!   r"   r#   r$   ray.util.placement_groupr%   r+   CreatePlacementGroupFnr8   strr>   rC   rF   rI   rK   rj   rt   CreateRouterCallabler   r   r   r   r)   r)   r)   r*   <module>   sn     $	 	


6