o `۷i6O@shddlZddlZddlZddlZddlmZddlmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZmZmZmZddlZddlZddlmZmZmZddlmZeeZ Gd d d Z!d a"da#d e$fd dZ%ddZ&deefddZ'de defddZ(Gddde)Z*dee e+fd e fddZ,GdddZ-edd d ed!fd"d#Z.d$e+fd%d&Z/d'e d(e fd e+fd)d*Z0d'e d(e fd e+fd+d,Z1d-ee+e d(e ffd.ee+e d(e fffd/d0Z2d-ee+e d(e ffd.ee+e d(e ffd e+fd1d2Z3d-ee+e d(e ffd.ee+e d(e ffd e+fd3d4Z4d5d6Z5d7d8Z6d9d:Z7d;d<Z8d=d>Z9dS)?N)contextmanager)wraps) Parameter) ModuleType) AnyCallableDict GeneratorListMutableMappingOptionalSequenceUnioncast)is_class_methodis_function_or_methodis_static_method)get_runtime_contextc@sPeZdZdZhdZddZddZddZd d Zd d Z d dZ ddZ dS)_OpenTelemetryProxya This proxy makes it possible for tracing to be disabled when opentelemetry is not installed on the cluster, but is installed locally. The check for `opentelemetry`'s existence must happen where the functions are executed because `opentelemetry` may be present where the functions are pickled. This can happen when `ray[full]` is installed locally by `ray` (no extra dependencies) is installed on the cluster. >traceContextcontext propagatecCs*|tjvrt|d|Std|)N_zAttribute does not exist: )rallowed_functionsgetattrAttributeError)selfnamerU/home/ubuntu/vllm_env/lib/python3.10/site-packages/ray/util/tracing/tracing_helper.py __getattr__/s z_OpenTelemetryProxy.__getattr__cC |dS)Nzopentelemetry.trace _try_importrrrr _trace5 z_OpenTelemetryProxy._tracecCr")Nzopentelemetry.contextr#r%rrr _context8r'z_OpenTelemetryProxy._contextcCr")Nzopentelemetry.propagater#r%rrr _propagate;r'z_OpenTelemetryProxy._propagatecCs|}|r |jjSdSN)r(rr)rrrrr _Context>sz_OpenTelemetryProxy._ContextcCs$||||dSr*)r&r(r)r+r%rrr try_allEs z_OpenTelemetryProxy.try_allcCs0zt|WStytrtdYdSw)NaInstall OpenTelemetry with 'pip install opentelemetry-api==1.34.1 opentelemetry-sdk==1.34.1 opentelemetry-exporter-otlp==1.34.1' to enable tracing. See the Ray documentation for details: https://docs.ray.io/en/latest/ray-observability/user-guides/ray-tracing.html#installation) importlib import_module ImportError_is_tracing_enabled)rmodulerrr r$Ks  z_OpenTelemetryProxy._try_importN) __name__ __module__ __qualname____doc__rr!r&r(r)r+r,r$rrrr r"s  rFreturncCstS)zgChecks environment variable feature flag to see if tracing is turned on. Tracing is off by default.)_global_is_tracing_enabledrrrr r0\sr0cCsdatatdS)NT)r7r_opentelemetryr,rrrr _enable_tracingbs r9 params_listcCs8t|D]\}}|jtjkr||||Sq|S)z]Given a list of Parameters, if a kwargs Parameter exists, move it to the end of the list.) enumeratekindr VAR_KEYWORDappendpop)r:iparamrrr _sort_params_listis rBfunction new_paramcsPt|}t|j}tfdd|Dr|St|g}|j|d}|S)z/Add additional Parameter to function signature.c3s|] }|jjkVqdSr*)r).0rArDrr xsz*_add_param_to_signature..) parameters)inspect signaturelistrHvaluesanyrBreplace)rCrDold_sigold_sig_list_repr new_paramsnew_sigrrFr _add_param_to_signaturess  rSc@s eZdZdS)_ImportFromStringErrorN)r2r3r4rrrr rTsrT import_strc Cst|ts|S|d\}}}|r|sd}t|j|dzt|}WntyB}z|j|kr4|dd}t|j|dd}~ww|}z| dD]}t ||}qKW|St ygd}t|j||d w) zRGiven a string that is in format ":", import the attribute.:zEImport string "{import_str}" must be in format":".)rUNz'Could not import module "{module_str}".) module_str.z;Attribute "{attrs_str}" not found in module "{module_str}".) attrs_strrW) isinstancestr partitionrTformatr-r.r/rsplitrr) rUrWrrYmessager1excinstanceattr_strrrr _import_from_strings8     rcc@s<eZdZdeeeffddZdeeefddfddZdS) _DictPropagatorr6cCsi}tj||S)z*Inject trace context into otel propagator.)r8rinject context_dictrrr inject_current_contexts z&_DictPropagator.inject_current_contextrg_opentelemetry.ContextcCsttjtj|S)z,Given a trace context, extract as a Context.)rr8rrextractrfrrr rjsz_DictPropagator.extractN)r2r3r4rrrhrjrrrr rdsrdparent_contextri)NNNc csL|dur|}nt}tj|}z dVWtj|dStj|w)z(Uses the Ray trace context for the span.N)r8rrattachdetach)rk new_contexttokenrrr _use_contexts rp function_namecCsvt}d|tt||d}tjjj j tjjj kr(| }|r(||d<t tjjj dd}|r9||d<|S)zXGet the Attributes of the function that will be reported as attributes in the trace.rC) ray.remote ray.functionray.pid ray.job_id ray.node_idz ray.task_id worker_idN ray.worker_id)rr[osgetpid get_job_id get_node_idray_privateworker global_workermode WORKER_MODE get_task_idrhex)rqruntime_context span_argstask_idrwrrr _function_hydrate_span_argss   rfunc.cC |dS)z>Returns the function span name that has span kind of producer. ray.remoterrrrr _function_span_producer_name rcCr)z>Returns the function span name that has span kind of consumer. ray.remote_workerrrrrr _function_span_consumer_namerrclass_methodcCst|r|j}t|r|j}t}d|||d|tt||d}tj j j j tj j j kr=|}|r=||d<ttj j j dd}|rN||d<|S)zUGet the Attributes of the actor that will be reported as attributes in the trace.actorrX)rrzray.actor_classzray.actor_methodrsrtrurv ray.actor_idrwNrx)callabler2rr[ryrzr{r|r}r~rrrr get_actor_idrr)rrrractor_idrwrrr _actor_hydrate_span_argss*    rcC0t|ts|j}t|ts|j}|d|dS)z;Returns the actor span name that has span kind of producer.rXrrZr[r2rrrrr _actor_span_producer_name   rcCr)z;Returns the actor span name that has span kind of consumer.rXrrrrrr _actor_span_consumer_namerrc s<t  d dtdtttfdtdtdtf fdd }|S) zfTrace the execution of a remote task. Inject the current span context into kwargs for propagation.Nargskwargs_args_kwargsr6cstr|jr|durd|vsJ|||g|Ri|Sd|vs#Jtjt}|jt|jtjj j t |jdt |d<|||g|Ri|WdS1sXwYdS)N_ray_trace_ctxr< attributes)r0_is_cross_languager8r get_tracerr2start_as_current_spanr_function_nameSpanKindPRODUCERrrdrh)rrrrrtracerrrr _invocation_remote_span)s     $z9_tracing_task_invocation.._invocation_remote_spanNN)rrr )rrrrr _tracing_task_invocation%s rc sbtsSttjdtjjdd_tdddtdtt t tfdtdtffdd }|S) zWrap the function argument passed to RemoteFunction's __init__ so that future execution of that function will include tracing. Use the provided trace context from kwargs. rNdefaultrrrr6c s|dur |i|Stjt}jdj}tt|6|jt |tjj j t |d|i|WdWdS1sIwYWddS1sYwYdS)NrXr) r8rrr2r3rprdrjrrrCONSUMERr)rrrrrqrCrr _function_with_tracingUs  Rz=_inject_tracing_into_function.._function_with_tracing) r0rSrIr KEYWORD_ONLY __signature__rrr rr[)rCrrrr _inject_tracing_into_functionFs( rc s:ttdfdtdtttfdtdtffdd }|S)z`Trace the creation of an actor. Inject the current span context into kwargs for propagation.Nrrrrc s|duri}tsd|vsJ|||g|Ri|S|jj}d}d|vs(Jtjt}|jt||tjj j t ||d&}t |d<|||g|Ri|} |d| j| WdS1shwYdS)Nr__init__rr<rr)r0__ray_metadata__ class_namer8rrr2rrrrrrdrh set_attribute _ray_actor_idr) rrrrrr method_namerspanresultrrr #_invocation_actor_class_remote_spanrs(    $zD_tracing_actor_creation.._invocation_actor_class_remote_span)rtuplerr )rrrrr _tracing_actor_creationns !rc s@t  d dttdtttfdtdtdtf fdd }|S) z(Trace the invocation of an actor method.Nrrrrr6c str|jjr|durd|vsJ|||g|Ri|S|jjj}|j}d|vs,Jtjt }|j t ||tjj j t||d%}t|d<|d|jj|||g|Ri|WdS1skwYdS)Nrrr)r0_actor_ray_is_cross_language'_ray_actor_creation_function_descriptorr _method_namer8rrr2rrrrrrdrhrrr) rrrrrrrrrrrr _start_spans$      $z5_tracing_actor_method_invocation.._start_spanr)rr rr )rrrrr _tracing_actor_method_invocations rcCsdtdtfdtfdd}dtdtfdtfdd}t|t}|D]W\}}t||s/t|r0q"t|s:t|r;q"|dkr@q"t |}t |tj d tj j d d |_ t|d d rZq"t|rht|||}nt|||}d|_t|||q"|S)zZGiven a class that will be made into an actor, inject tracing into all of the methods.r.r6c 8dddtdtdttttfdtdtf fdd}|S) Nrrrrrr6c str|dur|g|Ri|Stjt}tt|@|jt |j jtjj j t |j jd|g|Ri|WdWdS1sRwYWddS1sbwYdSzr Wrap the user's function with a function that will extract the trace context Nrr0r8rrr2rprdrjrr __class__rrrrrrrrrrr _resume_spans    RzF_inject_tracing_into_class..span_wrapper.._resume_spanrr rr[rrrrr span_wrapperz0_inject_tracing_into_class..span_wrapperc r) Nrrrrrr6c str|dur|g|Ri|IdHStjt}tt|E|jt |j jjtjj j t |j jjd|g|Ri|IdHWdWdS1s[wYWddS1skwYdSrrrrrr rs"    RzL_inject_tracing_into_class..async_span_wrapper.._resume_spanrrrrr async_span_wrapperrz6_inject_tracing_into_class..async_span_wrapper__del__rNr__ray_tracing_wrapped__FT)rrrI getmembersrrrisgeneratorfunctionisasyncgenfunctionunwraprSrrrriscoroutinefunctionrrsetattr)_clsrrmethodsrrunwrapped_methodwrapped_methodrrr _inject_tracing_into_classs2      r):r-rIloggingry contextlibr functoolsrrtypesrtypingrrrr r r r r rrr}ray._private.workerray._private.inspect_utilrrrray.runtime_contextr getLoggerr2loggerrr7r8boolr0r9rBrS ExceptionrTr[rcrdrprrrrrrrrrrrrrrr sr    0   6  "  $    !() %