o cia@sddlZddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z m Z m Z mZmZmZmZddlZddlZddlmZddlZddlmmZddlmZddlmZddlmZdd l m!Z!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,dd l-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5ddl6m7Z7m8Z8dZ9dZ:Gddde0Z;GdddZGdddZ?GdddZ@Gd d!d!ZAGd"d#d#ZBGd$d%d%ZCejDGd&d'd'ZEeEFZGd(d)ZHd*d+ZIe&fd,eJd-e#d.eKfd/d0ZLe&fd1eJd2eJd.eMfd3d4ZNe&fd,eJd5eMd2eJd.eMfd6d7ZOe&d8fd,eJd5eMd2eJd9eKd.eMf d:d;ZPe&fd,eJd5eMd2eJd.eMfdefd?d@ZR  ddAedBe"dCeeMdDeeee*eMe ffdEdFZSejTe:e'ddGGdHdIdIZUejDdJdidKGdLdMdMZVeVFZWdNdOZXe:fdPe.dQe dReJfdSdTZYddUdVZZddWdXZ[ddYdZZ\d[d\Z]d]d^Z^d_d`Z_dadbZ`edcefdddeZaGdfdgdgZbGdhdidiZcGdjdkdkZdejTdldmZed5eMfdndoZfe&dfd1eJd2eJdpefdqdrZgejTGdsdtdtZhddveJdweJfdxdyZie$jje&d8d8d8fdzeeJe$fd2eJd{eKd|eKd}eKd.eeJf d~dZke$jje&d8d8d8fdzeeJe$fd2eJd{eKd|eKd}eKd.eJf ddZldS)N)asynccontextmanager)copydeepcopy)AnyCallableDictListOptionalTupleUnion)Request)serve) ActorHandle)ServeControllerClient)CreatePlacementGroupRequest DeploymentIDDeploymentStatusRequestProtocol)SERVE_DEFAULT_APP_NAMESERVE_NAMESPACE)ALL_REPLICA_STATES ReplicaState)DRAINING_MESSAGE) ServeUsageTag) TimerBase)_get_global_client) serve_pb2serve_pb2_grpc)ApplicationStatus TargetGroupz /telemetrystoragec@s^eZdZddeefddZddeefddZdefdd Zd efd d Zd efddZ dS) MockTimerN start_timecCst|_|j|ddS)Nr") threadingLock_lockresetselfr"r*Q/home/ubuntu/.local/lib/python3.10/site-packages/ray/serve/_private/test_utils.py__init__+s zMockTimer.__init__cCs|durt}||_dSN)time_currr(r*r*r+r'/s zMockTimer.resetreturncC|jSr-r/r)r*r*r+r.4zMockTimer.timebycCs:|j|j|7_WddS1swYdSr-r&r/)r)r5r*r*r+advance7s"zMockTimer.advanceamtcCs>|j|j|d7_WddS1swYdS)NgMbP?r6r)r8r*r*r+realistic_sleep;s"zMockTimer.realistic_sleepr-) __name__ __module__ __qualname__r floatr,r'r.r7r:r*r*r*r+r!*s r!c@s\eZdZddeefddZdddZdefdd Zd efd d Zd efd dZ ddZ dS)MockAsyncTimerrr"cCs|j|dd|_dS)Nr#r)r' _num_sleepersr(r*r*r+r,As  zMockAsyncTimer.__init__cC ||_dSr-r2r(r*r*r+r'E zMockAsyncTimer.resetr0cCr1r-r2r3r*r*r+r.Hr4zMockAsyncTimer.timer8csP|jd7_|j|}|j|krtdIdH|j|ks|jd8_dS)Nr)r@r/asynciosleep)r)r8endr*r*r+rEKs   zMockAsyncTimer.sleepcCs|j|7_dSr-r2r9r*r*r+r7UzMockAsyncTimer.advancecCr1r-)r@r3r*r*r+ num_sleepersXr4zMockAsyncTimer.num_sleepersN)r)r"r) r;r<r=r r>r,r'r.rEr7rHr*r*r*r+r?@s  r?c@sNeZdZddZdededefddZdedefdd Zdedefd d Z d S) MockKVStorecCs t|_dSr-)dictstorer3r*r*r+r,]s zMockKVStore.__init__keyvalr0cCs*t|tstdt|||j|<dS)Nkey must be a string, got: {}.T isinstancestr TypeErrorformattyperK)r)rLrMr*r*r+put`s  zMockKVStore.putcCs*t|tstdt||j|dS)NrN)rPrQrRrSrTrKgetr)rLr*r*r+rVfs zMockKVStore.getcCs6t|tstdt|||jvr|j|=dSdS)NrNTFrOrWr*r*r+deleteks  zMockKVStore.deleteN) r;r<r=r,rQrboolrUrVrXr*r*r*r+rI\s rIc@sneZdZddZddZddZddZd d Zd d Zd dZ dde de de fddZ de de fddZ dS)MockClusterNodeInfoCachecCs,t|_t|_t|_t|_t|_dSr-)setalive_node_idsrJtotal_resources_per_nodeavailable_resources_per_nodedraining_nodes node_labelsr3r*r*r+r,ws  z!MockClusterNodeInfoCache.__init__cCr1r-)r\r3r*r*r+get_alive_node_ids~r4z+MockClusterNodeInfoCache.get_alive_node_idscCr1r-)r_r3r*r*r+get_draining_nodesr4z+MockClusterNodeInfoCache.get_draining_nodescCs|jt|jSr-)r\r[r_r3r*r*r+get_active_node_idsz,MockClusterNodeInfoCache.get_active_node_idscCdSr-r*)r)node_idr*r*r+ get_node_azz$MockClusterNodeInfoCache.get_node_azcCr1r-)r^r3r*r*r+ get_available_resources_per_noder4z9MockClusterNodeInfoCache.get_available_resources_per_nodecCr1r-)r]r3r*r*r+get_total_resources_per_noder4z5MockClusterNodeInfoCache.get_total_resources_per_nodeNrf resourceslabelscCsB|j|t|p i|j|<t|pi|j|<|pi|j|<dSr-)r\addrr]r^r`)r)rfrkrlr*r*r+add_nodes z!MockClusterNodeInfoCache.add_nodecCst||j|<dSr-)rr^)r)rfrkr*r*r+ set_available_resources_per_noderGz9MockClusterNodeInfoCache.set_available_resources_per_nodeNN)r;r<r=r,rarbrcrgrirjrQrrnror*r*r*r+rZvsrZc@eZdZddZdS)FakeRemoteFunctioncCrer-r*r3r*r*r+remoterhzFakeRemoteFunction.remoteN)r;r<r=rsr*r*r*r+rrs rrc@s,eZdZddZeddZeddZdS)MockActorHandlecKs||_d|_d|_d|_dS)Nfake_idF)_options _actor_id"initialize_and_get_metadata_calledis_allocated_called)r)kwargsr*r*r+r,s zMockActorHandle.__init__cC d|_tSNT)rxrrr3r*r*r+initialize_and_get_metadatasz+MockActorHandle.initialize_and_get_metadatacCr{r|)ryrrr3r*r*r+ is_allocatedszMockActorHandle.is_allocatedN)r;r<r=r,propertyr}r~r*r*r*r+rts  rtc@s*eZdZddZddZdefddZdS) MockActorClasscCd|_t|_dSNr*) _init_argsrJrvr3r*r*r+r, zMockActorClass.__init__cKs(t|}|D] \}}||j|<q|Sr-)ritemsrv)r)rzreskvr*r*r+optionss zMockActorClass.optionsr0cGstdd|i|jS)N init_argsr*)rtrv)r)argsr*r*r+rszMockActorClass.remoteN)r;r<r=r,rrtrsr*r*r*r+rsrc@seZdZdefddZdS)MockPlacementGrouprequestcCs*|j|_|j|_|j|_|j|_d|_dS)Ndetached) bundles_bundlesstrategy _strategytarget_node_id_soft_target_node_idname_name _lifetime)r)rr*r*r+r,s  zMockPlacementGroup.__init__N)r;r<r=rr,r*r*r*r+rsrc@s~eZdZefdedefddZddZddZd d Zd e efd d Z de fddZ ddZ defddZdefddZdS)MockDeploymentHandledeployment_nameapp_namecCs$||_||_tj|_d|_d|_dS)NF)_deployment_name _app_namer UNDEFINED _protocol_running_replicas_populated _initialized)r)rrr*r*r+r,s  zMockDeploymentHandle.__init__cCr1r-)rr3r*r*r+is_initializedr4z#MockDeploymentHandle.is_initializedcCs|jrtdd|_dS)Nzalready initializedT)r RuntimeErrorr3r*r*r+_inits zMockDeploymentHandle._initcOs|Sr-r*)r)rrzr*r*r+rrhzMockDeploymentHandle.optionsdepcCs|\}}|j|ko |j|kSr-)rr)r)rother_deployment_nameother_app_namer*r*r+__eq__s zMockDeploymentHandle.__eq__protocolcCrAr-)r)r)rr*r*r+_set_request_protocolrBz*MockDeploymentHandle._set_request_protocolcCrer-r*r3r*r*r+_get_or_create_routerrhz*MockDeploymentHandle._get_or_create_routerr0cCr1r-rr3r*r*r+running_replicas_populatedr4z/MockDeploymentHandle.running_replicas_populatedrMcCrAr-r)r)rMr*r*r+set_running_replicas_populatedrBz3MockDeploymentHandle.set_running_replicas_populatedN)r;r<r=rrQr,rrrr rrrrrYrrr*r*r*r+rsrc@rq)GetPIDcCstSr-)osgetpidr3r*r*r+__call__szGetPID.__call__N)r;r<r=rr*r*r*r+rs rcCs&ztdWdStyYdSw)N%http://localhost:8265/api/ray/versionFT)requestsrV Exceptionr*r*r*r+check_ray_stoppeds   rcCstdjdkS)Nr)rrV status_coder*r*r*r+check_ray_startedrdrrexpected_statusr0cCs&tj|}|j|j|ksJdSr|)r status applications deployments)rrr app_statusr*r*r+check_deployment_status srrrcCs.t||d}tjdd|fdgd}t|S)z= target.Trrrrr*r*r+check_num_replicas_gte rFuse_controllercCsN|rtj|j|}|jtjd}||ksJdSt|||ks%JdS)z#Check if num replicas is == target.rT) r rrrreplica_statesrVrRUNNINGr)rrrrrnum_running_replicasr*r*r+check_num_replicas_eq)s rcCst|||ks JdS)z#Check if num replicas is <= target.Trrr*r*r+check_num_replicas_lte;rrappscCs,t}|D] }|j|jtjksJqdSr|)r rrrr)rrrr*r*r+check_apps_runningDsr controller deployment_idtotalby_statec st|j||dur fddtD}|ks J||durn|D]G\}}t|ts2Jt|tr;|dks=JrRfddj|gdD}t |}nj|gd}d|d |d |d } ||ksmJ| q&d S) aUses _dump_replica_states_for_testing to check replica counts. Args: controller: A handle to the Serve controller. deployment_id: The deployment to check replica counts for. total: The total number of expected replicas for the deployment. by_state: A list of tuples of the form (replica state, number of replicas, filter function). Used for more fine grained checks. Ncs*i|]}|gr|t|gqSr*)rVr).0r)replicasr*r+ bs  z(check_replica_counts..rcsh|]}|r|qSr*r*)rr)checkr*r+ nz'check_replica_counts..)statesz Expected z for state z but got .T) rayrV _dump_replica_states_for_testingrsrcountrPrintr) rrrrreplica_countsrrfiltered curr_countmsgr*)rrr+check_replica_countsMs&   r)r namespacenum_cpusc@sBeZdZddZdeddfddZdefdd Zdefd d ZdS) TelemetryStoragecCrNr)reports_receivedrJcurrent_reportr3r*r*r+r,zrzTelemetryStorage.__init__reportr0NcCs|jd7_||_dSNrC)rr)r)rr*r*r+ store_report~s zTelemetryStorage.store_reportcCr1r-)rr3r*r*r+ get_reportr4zTelemetryStorage.get_reportcCr1r-)rr3r*r*r+get_reports_receivedr4z%TelemetryStorage.get_reports_received) r;r<r=r,rrrrrr*r*r*r+rxs rr)ray_actor_optionsc@s&eZdZddZdedefddZdS)TelemetryReceivercCstjttd|_dS)N)rr)r get_actorSTORAGE_ACTOR_NAMErr r3r*r*r+r,rzTelemetryReceiver.__init__rr0cs(|IdH}t|jj|dSr|)jsonrrVr rrs)r)rrr*r*r+rszTelemetryReceiver.__call__N)r;r<r=r,r rYrr*r*r*r+rsrcCst}tjtdtd|S)uStart a telemetry Serve app. Ray should be initialized before calling this method. NOTE: If you're running the TelemetryReceiver Serve app to check telemetry, remember that the receiver itself is counted in the telemetry. E.g. if you deploy a Serve app other than the receiver, the number of apps in the cluster is 2- not 1– since the receiver is also running. Returns a handle to a TelemetryStorage actor. You can use this actor to access the latest telemetry reports. telemetry)r route_prefix)rrsr run receiver_appTELEMETRY_ROUTE_PREFIX)r r*r*r+start_telemetry_appsrtagexpectedstorage_actor_namecCs@tj|td}t|j}t|d|||ksJdS)N)rextra_usage_tagsT)rrrrVrrsprintget_value_from_report)rrrstorage_handlerr*r*r+check_telemetrys  r c Csddl}t|}t}|rE|tj}|jj |d\}}Wdn1s)wY|j }| tj j ks;J|tksCJdS|jj |d\} } | tj jksXJ| j|ks_JdS)NrrT)pytestrRayServeAPIServiceStubrListApplicationsRequestraisesgrpcRpcErrorListApplications with_callvaluecode StatusCode UNAVAILABLEdetailsrOKapplication_names) channel app_names test_drainingrstubrexception_info_ rpc_errorresponsecallr*r*r+ping_grpc_list_applicationss r&c Csddl}t|}t}|rE|tj}|jj |d\}}Wdn1s)wY|j }| tj j ks;J|tksCJdS|jj |d\}} | tj jksXJ|jdks_JdS)Nrr success)rrrrHealthzRequestrrrHealthzrrrrrrrrmessage) rrrr rr!r"r#r$r%r*r*r+ping_grpc_healthzs r+c Csddl}t|}tjdddd}d|ff}|rW|tj}|jj ||d\}}Wdn1s3wY|j } | tj j ksIJ| d|d | vsUJdS|jj ||d\} } | tj jksoJ| | jd ksyJ| jdS) Nrfoobarrnumr, applicationrmetadataz Application 'z ' not found.zHello foo from bar)rrUserDefinedServiceStubrUserDefinedMessagerrrrrrrr NOT_FOUNDrrgreeting) rrtest_not_foundrr rr3r!r"r#r$r%r*r*r+ping_grpc_call_methods  r9cCsDt|}tjdddd}d|ff}|j||d}|jdks JdS)Nr,r-r.r/r1r2zHello foo from method1)rr4rr5Method1r7rrr rr3r$r*r*r+ping_grpc_another_methods  r<cCsTt|}tjdddd}d}d|fd|ff}|j||d}|jd |ks(JdS) Nr,r-r.r/999r1multiplexed_model_idr2z%Method2 called model, loading model: )rr4rr5Method2r7)rrr rr>r3r$r*r*r+ping_grpc_model_multiplexings  r@cCs\t|}tjdddd}d|ff}|j||d}t|D]\}}|j|dks+JqdS)Nr,r-r.r/r1r2z: Hello foo from bar)rr4rr5 Streaming enumerater7)rrr rr3 responsesidxr$r*r*r+ping_grpc_streamings  rEcCsBt|}tjddd}d|ff}|j||d}|jdksJdS)N)orangeappler1r2 )rFruitServiceStubr FruitAmounts FruitStandcostsr;r*r*r+ping_fruit_stand s  rO signal_actorcCshd}z dVtdIdHWntjy+d}tjjd|jIdHYnw|s2t ddS)NFdTzGCancelledError wasn't raised during `send_signal_on_cancellation` block) rDrECancelledErrorr_raylet async_task_idr[sendrsr)rP cancelledr*r*r+send_signal_on_cancellationsrWc@sleZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZdS)FakeGrpcContextcCs@ddi|_dg|_d|_d|_d|_d|_d|_g|_g|_dS)NrLr)rLrpeerspeer_identitiespeer_identity_key) _auth_context_invocation_metadata_peer_peer_identities_peer_identity_key_code_details_trailing_metadatar3r*r*r+r,$s  zFakeGrpcContext.__init__cCr1r-)r[r3r*r*r+ auth_context/r4zFakeGrpcContext.auth_contextcCr1r-r`r3r*r*r+r2r4zFakeGrpcContext.codecCr1r-rar3r*r*r+r5r4zFakeGrpcContext.detailscCr1r-)r]r3r*r*r+rY8r4zFakeGrpcContext.peercCr1r-)r^r3r*r*r+peer_identities;r4zFakeGrpcContext.peer_identitiescCr1r-)r_r3r*r*r+rZ>r4z!FakeGrpcContext.peer_identity_keycCr1r-rbr3r*r*r+trailing_metadataAr4z!FakeGrpcContext.trailing_metadatacCrAr-rd)r)rr*r*r+set_codeDrBzFakeGrpcContext.set_codecCrAr-re)r)rr*r*r+ set_detailsGrBzFakeGrpcContext.set_detailscCrAr-rg)r)rhr*r*r+set_trailing_metadataJrBz%FakeGrpcContext.set_trailing_metadatacCr1r-)r\r3r*r*r+invocation_metadataMr4z#FakeGrpcContext.invocation_metadataN)r;r<r=r,rcrrrYrfrZrhrirjrkrlr*r*r*r+rX#s  rXc@steZdZddedeefddZdeeeffddZdd ee e fdeeeffd d Z deeeffd d Z dS) FakeGaugeNrtag_keyscC$||_t|_|p d|_t|_dSr)rrJvaluestags default_tagsr)rrnr*r*r+r,R  zFakeGauge.__init__rqcC.|D]\}}||jvsJ||j|<qdSr-rrqrrr)rqrLrr*r*r+set_default_tagsY zFakeGauge.set_default_tagsrcCs|j}||p it|t|jksJ|j}|jddD]}||}||vr2t||<||}q#||||jd<dS)N)rrrupdater[keysrqrprJ)r)rrq merged_tagsdr tag_valuer*r*r+r[^s   z FakeGauge.setcC6|j}|jD]}||}||}|durdSq|Sr-)rprqrVr)rqrrrr*r*r+ get_valuel  zFakeGauge.get_valuerpr-) r;r<r=rQr r,rrxr rr>r[rr*r*r*r+rmQs $rmc@seZdZddedeefddZdeeeffddZdd ee e fdeeeffd d Z deeefd e fddZ ddZ dS) FakeCounterNrrncCror)rrJcountsrqrrrsr*r*r+r,xrtzFakeCounter.__init__rqcCrur-rvrwr*r*r+rxryzFakeCounter.set_default_tags?rcCs|j}||p it|t|jksJ|j}|jddD]}||}||vr2t||<||}q#||jd}||d|||<dS)Nrzr) rrrr{r[r|rqrrJrV)r)rrqr}r~rrrLr*r*r+incs   zFakeCounter.incr0cCrr-)rrqrVrr*r*r+ get_countrzFakeCounter.get_countcCr1r-)rqr3r*r*r+get_tagsr4zFakeCounter.get_tagsrp)rN)r;r<r=rQr r,rrxr rr>rrrr*r*r*r+rws $ rcCs tSr-)rget_runtime_context get_node_idr*r*r*r+rs rcCs&ddtD}t||ksJdS)NcSsg|]}|dr|qS)Aliver*)rnoder*r*r+ rz)check_num_alive_nodes..T)rnodesr)r alive_nodesr*r*r+check_num_alive_nodessr_clientcCs&|pt}|}|d|d|S)Nrr)rget_serve_details)rrrclientrr*r*r+get_deployment_detailss rc@s*eZdZdefddZddZddZdS) CounterrcCsd|_||_t|_dSr)rrrDEvent ready_event)r)rr*r*r+r,szCounter.__init__cCs,|jd7_|j|jkr|jdSdSr)rrrr[r3r*r*r+rs z Counter.inccs|jIdHdSr-)rwaitr3r*r*r+rsz Counter.waitN)r;r<r=rr,rrr*r*r*r+rs rINFOslevelcCs6tjddd}td|d|d|dS)z&Convenient logging method for testing.z %H:%M:%S.%fN[z]  )datetimenowstrftimer )rrrr*r*r+tlogsrr use_localhost is_websocketexclude_route_prefixcs@t}|}||dvr|jgS|d|d}|rd}ttr&tt|jj |}fdd|D}t |dkrLt d|dd |g} |D]M} | j D]G} |r[d n| j} tjkrw|rgd nd } | d | d| j|}ntjkr|rt d| d| j}nt d|d}| |qUqP| S)a Get the URL of the application. Args: protocol: The protocol to use for the application. app_name: The name of the application. use_localhost: Whether to use localhost instead of the IP address. Set to True if Serve deployments are not exposed publicly or for low latency benchmarking. is_websocket: Whether the url should be served as a websocket. exclude_route_prefix: The route prefix to exclude from the application. Returns: The URLs of the application. rrcsg|] }|jkr|qSr*r)r target_grouprr*r+rs  z(get_application_urls..rzNo target group found for app z with protocol z and route prefix localhostwshttpz://:z6is_websocket=True is not supported with gRPC protocol.zUnsupported protocol: /)rrroot_urlrPrQrrrV _controllerget_target_groupsrsr ValueErrortargetsipHTTPportGRPCrstripappend)rrrrrr serve_detailsr target_groupsurlsrrrschemeurlr*rr+get_application_urlssJ           rcCstt|||||S)aUGet the URL of the application. Args: protocol: The protocol to use for the application. app_name: The name of the application. use_localhost: Whether to use localhost instead of the IP address. Set to True if Serve deployments are not exposed publicly or for low latency benchmarking. is_websocket: Whether the url should be served as a websocket. exclude_route_prefix: The route prefix to exclude from the application. Returns: The URL of the application. If there are multiple URLs, a random one is returned. )randomchoicer)rrrrrr*r*r+get_application_url s  rrp)F)r)mrDrrrr$r. contextlibrrrtypingrrrrr r r rrstarlette.requestsr rray.util.stateutilrrr ray.actorrray.serve._private.clientrray.serve._private.commonrrrrray.serve._private.constantsrr#ray.serve._private.deployment_staterrray.serve._private.proxyrray.serve._private.usagerray.serve._private.utilsrray.serve.contextrray.serve.generatedrrray.serve.schemarrrrr!r?rIrZrrrtrrr deploymentrbindget_pid_entrypointrrrQrYrrrrrrrrrsrrrrr r&r+r9r<r@rErOrWrXrmrrrrrrrrrr*r*r*r+sh $        $ (          +    .&*     ?