o
    i}Z                     @   sB  d dl mZ d dlmZ d dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' ddl(m)Z) ddl(m*Z* ddl(m+Z+ ddl(m,Z, ddl(m-Z- ddl(m.Z. ddl(m/Z/ ddl(m0Z0 ddl(m1Z1 ddl(m2Z2 ddl(m3Z3 dd l(m4Z4 dd!l(m5Z5 dd"l(m6Z6 dd#l(m7Z7 dd$l(m8Z8 dd%l(m9Z9 dd&l(m:Z: dd'l(m;Z; dd(l(m<Z< dd)l(m=Z= dd*l(m>Z> dd+l(m?Z? dd,l@mAZA dd-l@mBZB dd.l@mCZC dd/lDmEZE dd0lDmFZF dd1lDmGZG dd2lDmHZH dd3lDmIZI dd4lDmJZJ dd5lDmKZK dd6lDmLZL dd7lDmMZM dd8lDmNZN dd9lDmOZO eePZQejRSe/ZTd:hZUh eUZVh eUd;d:ZWeXd<eYed<e%ejZd=d>d?e%ejZd@dAd?e!dBd>e%dCe!dDd>e%dCdE dFeYe[e[f fdGdHZ\dFe[fdIdJZ]dKdL Z^dMdN Z_dOdP Z`dQdR ZadSdT ZbdUdV ZcdWdX ZddYdZ Zed[e	d\ef dFefd]d^Zfd[e	d\ef dFefd_d`Zgedaed[e	d\ef fdbdcZhd[e	d\ef dFefdddeZid[e	d\ef dFefdfdgZjdhdi Zkdjdk Zldldm Zmdndo ZndS )p    )contextmanagerwrapsNAny)Callable)wrap_function_wrapper)configtracer)	SPAN_KIND)unwrap)SpanKind	SpanTypes)
get_logger)ModuleWatchdog)schematize_service_name)
get_config)get_argument_value)asbool)_TraceContext   )DD_RAY_TRACE_CTX)DEFAULT_JOB_NAME)RAY_ACTOR_METHOD_ARGS)RAY_ACTOR_METHOD_KWARGS)RAY_ENTRYPOINT)RAY_GET_VALUE_SIZE_BYTES)RAY_JOB_NAME)RAY_JOB_STATUS)RAY_JOB_SUBMIT_STATUS)RAY_PUT_VALUE_SIZE_BYTES)RAY_PUT_VALUE_TYPE)RAY_STATUS_ERROR)RAY_STATUS_FAILED)RAY_STATUS_SUCCESS)RAY_SUBMISSION_ID)RAY_SUBMISSION_ID_TAG)RAY_TASK_ARGS)RAY_TASK_KWARGS)RAY_TASK_STATUS)RAY_TASK_SUBMIT_STATUS)RAY_WAIT_FETCH_LOCAL)RAY_WAIT_NUM_RETURNS)RAY_WAIT_TIMEOUT)long_running_ray_span)start_long_running_job)stop_long_running_job)ENTRY_POINT_REGEX)!_extract_tracing_context_from_env)_inject_context_in_env)_inject_context_in_kwargs)_inject_dd_trace_ctx_kwarg)!_inject_ray_span_tags_and_metrics)extract_signature)flatten_metadata_dict)get_dd_job_name_from_entrypoint)redact_paths)set_tag_or_truncatezray.data._internalzray.experimentalray+DD_TRACE_RAY_USE_ENTRYPOINT_AS_SERVICE_NAMEF)default$DD_TRACE_RAY_REDACT_ENTRYPOINT_PATHSTDD_TRACE_RAY_CORE_API)r@   modifierDD_TRACE_RAY_ARGS_KWARGS)_default_serviceuse_entrypoint_as_service_nameredact_entrypoint_pathstrace_core_apitrace_args_kwargsreturnc                   C   s   ddiS )Nr>   z>=2.46.0 rK   rK   rK   V/home/ubuntu/.local/lib/python3.10/site-packages/ddtrace/contrib/internal/ray/patch.py_supported_versions_   s   rM   c                   C   s   t ttddS )N__version__ )strgetattrr>   rK   rK   rK   rL   get_versionc   s   rR   c                    s   t   fdd}|S )z6Inject trace context parameter into function signaturec                     s   t  g| R i |S N)_wrap_task_execution)argskwargsfunctionrK   rL   wrapped_functionj   s   z9_wrap_remote_function_execution.<locals>.wrapped_functionr   )rX   rY   rK   rW   rL   _wrap_remote_function_executiong   s   rZ   c                 O   s   t |vr| |i |S t|t  }|t  td| j d| j ttj	|ddL}z't
jjr;t|t| t|t| | |i |}|tt |W W  d   S  tyt } ztjd| j d| j |dd |tt  d}~ww 1 sxw   Y  dS )zL
    Wraps the actual execution of a Ray task to trace its performance.
    ztask.execute.Tresourceservice	span_typechild_ofactivateNz Ray task %s execution failed: %sexc_info)r   r   _extractpopr0   
__module____qualname__RAY_SERVICE_NAMEr   RAYr	   r>   rI   r=   r)   r*   _set_tag_strr+   r&   BaseExceptionlogdebugr$   )wrappedrU   rV   extracted_contexttask_execute_spanresulterK   rK   rL   rT   q   s<   
rT   c                 C   s  |j jtv s	|jr| |i |S t du r$td|j tj	
t  |j  |jdu r@t|j |_ t|j |j _t|j |_W d   n1 sJw   Y  tjd|j dttjdn}|ttj t| z@tjjrt|t| di  t|t!| di  t"t#$|j j%v }|rt&|j'| | |i |}|t(t) |W W  d   S  t*y } ztjd|j d	|d
d |t(t+ |d}~ww 1 sw   Y  dS )z1Trace task submission, i.e the func.remote() callNzNNo active span found in %s.remote(), activating trace context from environmentztask.submit.remoter]   r^   r_   rU   rV   z!Failed to submit Ray task %s : %sz	.remote()Trb   ),	_functionrf   RAY_TASK_MODULE_DENYLIST_is_cross_languager   current_spanrl   rm   _function_namecontext_providerra   r4   _inject_lock_function_signaturerZ   r7   __signature__r9   tracerh   r   ri   rj   r   r   PRODUCERr8   r	   r>   rI   r=   r)   getr*   r   inspect	signature
parametersr6   contextr,   r&   rk   r$   )rn   instancerU   rV   spanhas_trace_ctxresprr   rK   rK   rL   traced_submit_task   sR   

r   c                 C   sX  ddl m} |dp| }||d< |dd}tjjr t|}|di p'i }tjjr3t|p1t	}ntj
r9tjnd}	|dd}
|	pF|
pFt	}tjd	|pMt	tjd
}zt| |t| |re|t| t|}| D ]
\}}t||| qmtj| t| tjd|pt	tjd
n}t| |ttj |t| |dpi }||d< |dpi }||d< t|j | ||t!< |r||t"< z| |i |}|t#t$ |W W  d   W S  t%y } zt&j'd||dd |t#t(  d}~ww 1 sw   Y  W dS  t%y+ } z|t)t( d|_*|+t,|||j- t.| |d}~ww )zTrace job submission. This function is also responsible
    of creating the root span.
    It will also inject _RAY_SUBMISSION_ID and _RAY_JOB_NAME
    in the env variable as some spans will not have access to them
    through ray_ctx
    r   )generate_job_idsubmission_id
entrypointrO   metadataNjob_namezray.jobr^   r_   zray.job.submitruntime_envenv_varsz Failed to submit Ray Job %s : %sTrb   r   )/%ray.dashboard.modules.job.job_managerr   r   r	   r>   rG   r<   rF   r;   r   _is_user_provided_servicer^   r   
start_spanr   ri   r8   rj   r(   r   r:   itemsr=   rz   ra   r1   r~   r   r   r   r   _injectr   r'   r   r!   r&   rk   rl   rm   r$   r    errorset_exc_infotype__traceback__r2   )rn   r   rU   rV   r   r   r   r   r   user_provided_servicemetadata_job_namejob_span	dot_pathskvsubmit_spanr   r   r   rr   rK   rK   rL   traced_submit_job   st   r   c              	   C   s  |j j}t||dd}tdd |j| D s| |i |S t du r2td|| tj	
t  tjdttj| d| d	d
:}|ttj tjjrdt|tt||dd t|tt||dd t| t|j| | |i |W  d   S 1 sw   Y  dS )zHTrace actor method submission, i.e the Actor.func.remote()
    call
    r   method_namec                 s   s    | ]}|j tkV  qd S rS   )namer   ).0prK   rK   rL   	<genexpr>      z+traced_actor_method_call.<locals>.<genexpr>NzQNo active span found in %s.%s.remote(), activating trace context from environmentzactor_method.submitr[   rs   )r^   r_   r]   rU   r   rV   )'_ray_actor_creation_function_descriptor
class_namer   any_ray_method_signaturesr   rx   rl   rm   rz   ra   r4   r~   rh   r   ri   rj   r   r   r   r	   r>   rI   r=   r   r   r8   r6   r   )rn   r   rU   rV   
actor_namer   r   rK   rK   rL   traced_actor_method_call  s0   $r   c                 C   s   t jjs| |i |S t du rtjt  tdt	pt
tjtj dd=}|ttj |d}|dur@|dt| t| t||dd}|ttt| | |i |W  d   S 1 sgw   Y  dS )	z$
    Trace the calls of ray.get
    Nzray.getTr^   r_   r`   ra   timeoutzray.get.timeout_sr   object_refs)r	   r>   rH   r   rx   rz   ra   r4   r0   rh   r   r   ri   activerj   r   r   r   r   rP   r8   r   r   sys	getsizeof)rn   r   rU   rV   r   r   	get_valuerK   rK   rL   
traced_get8  s*   
$r   c                 C   s   t jjs| |i |S t du rtjt  tjdt	pt
tjd7}|ttj t| t||dd}|ttt|j |ttt| | |i |W  d   S 1 s]w   Y  dS )z$
    Trace the calls of ray.put
    Nzray.putr   r   value)r	   r>   rH   r   rx   rz   ra   r4   r~   rh   r   r   ri   rj   r   r   r   r8   r   r#   rP   r   __name__r"   r   r   )rn   r   rU   rV   r   	put_valuerK   rK   rL   
traced_putS  s   $r   c                 C   s  t jjs| |i |S t du rtd tjt	  t
dttjtj ddM}|ttj t| |d}|d}|d}|durQ|tt| |dur]|tt| |duri|tt| | |i |W  d   S 1 szw   Y  dS )	z%
    Trace the calls of ray.wait
    NzMNo active span found in ray.wait(), activating trace context from environmentzray.waitTr   r   num_returnsfetch_local)r	   r>   rH   r   rx   rl   rm   rz   ra   r4   r0   rh   r   ri   r   rj   r   r   r   r8   r   r/   rP   r.   r-   )rn   r   rU   rV   r   r   r   r   rK   rK   rL   traced_waith  s4   



$r   method.c              	      (   d ddt dt dt dt f fdd}|S )N_dd_ray_trace_ctxselfrU   rV   rJ   c                   s  dd l }ddlm} |rt|nd }tjt}t	d| j
j d j tjtt|j|ddM}t|j z | g|R i |I d H  W n& |jjyn }	 z| j|I d H }
t|
jtkrd|	W Y d }	~	nd }	~	ww W d    d S W d    d S 1 sw   Y  d S )Nr   r   actor_method.executer[   Tr\   )ray.exceptionsddtrace.extr   r   rd   osenvironr   r'   r0   	__class__r   r   r   ri   r5   r   
exceptionsAsyncioActorExit_job_info_clientget_inforP   statusr%   )r   r   rU   rV   r>   r   r   r   supervisor_run_spanrr   job_infor   rK   rL   _traced_run_method  s8   
 "z7_job_supervisor_run_wrapper.<locals>._traced_run_methodr   )r   r   rK   r   rL   _job_supervisor_run_wrapper  s   $r   c              	      r   )Nr   r   rU   rV   rJ   c          	         s   ddl m} ddlm} t| j}|r|d d}ntj	
| j}| d u r7td| |jt  |jdd| tjtt|jd	}|ttj t|  | g|R i |W  d    S 1 sjw   Y  d S )
Nr   r
   r   r   z.pyzJNo active span found in exec %s, activating trace context from environmentzexec entrypointzexec rt   )ddtracer   r   r   r3   search_entrypointgroupr   pathbasenamerx   rl   rm   rz   ra   r4   r~   r   r   r   r   ri   rj   r   r   CONSUMERr8   )	r   r   rU   rV   r   r   script_name_matchentrypoint_namer   r   rK   rL   _traced_exec_entrypoint_method  s(   $z@_exec_entrypoint_wrapper.<locals>._traced_exec_entrypoint_methodr   )r   r   rK   r   rL   _exec_entrypoint_wrapper  s   $r   r   c                 o   s    t j }|d u rt|}td| jj d|j tt	j
|dd}tjjr3t|t| t|t| |V  W d    d S 1 sAw   Y  d S )Nr   r[   Tr\   )r   rz   r   r   rd   r0   r   r   rh   r   ri   r	   r>   rI   r=   r   r   )r   r   dd_trace_ctxrU   rV   r   actor_execute_spanrK   rK   rL   _trace_actor_method  s$   

"r   c              	      r   )Nr   r   rU   rV   rJ   c                   s   ddl m} |d u r| d u r | g|R i |S t|  |g|R i |  | g|R i |W  d    S 1 s>w   Y  d S Nr   r
   r   r   rx   r   r   r   rU   rV   r   r   rK   rL   _traced_method  s   $z4_inject_tracing_actor_method.<locals>._traced_methodr   )r   r   rK   r   rL   _inject_tracing_actor_method     $	r   c              	      r   )Nr   r   rU   rV   rJ   c                   s   ddl m} |d u r| d u r | g|R i |I d H S t|  |g|R i |  | g|R i |I d H W  d    S 1 sEw   Y  d S r   r   r   r   rK   rL   _traced_async_method  s   $z@_inject_tracing_async_actor_method.<locals>._traced_async_methodr   )r   r   rK   r   rL   "_inject_tracing_async_actor_method  r   r   c                    sv  ddl m} ddl m} ddl m} | |i |}t|j t|j}t fddtD r0|S |	dr7|S   d| d	k}	|	rFd
dhnt
 }
t||}|D ]g\}}||
v rZqQ|||sq||sqt|sqt|sq|dkrytd|| qQt||_|	r|dkrt|t|}n%|	r|dkrt|t|}nt|rt|t|}nt|t|}t||| qQ|S )Nr   )is_class_method)is_function_or_method)is_static_methodc                 3   s    | ]}  |V  qd S rS   )
startswith)r   denied_modulemodule_namerK   rL   r     r   z2inject_tracing_into_actor_class.<locals>.<genexpr>_r[   z6ray.dashboard.modules.job.job_supervisor.JobSupervisorping_polling__del__z/Skipping method %s.%s (unsupported method type)run_exec_entrypoint)ray._private.inspect_utilr   r   r   rP   rf   r   r   RAY_ACTOR_MODULE_DENYLISTr   setr   
getmembersisgeneratorfunctionisasyncgenfunctionrl   rm   r7   r}   r   r   r   iscoroutinefunctionr   r   setattr)rn   r   rU   rV   r   r   r   clsr   is_job_supervisormethods_to_ignoremethodsr   r   wrapped_methodrK   r   rL   inject_tracing_into_actor_class  sJ   




r  c                    sD   | |i |I d H }t ||dd}|j|I d H }t|| |S )Nr   job_id)r   r   r   r2   )rn   r   rU   rV   rq   r  r   rK   rK   rL   traced_end_job+  s   
r  c                  C   s   t tddrd S dt_ddlm}  d| _tddd }td	d
d }tddd }ttdt	 ttdt
 ttdt d S )N_datadog_patchFTr   )tracing_helperz	ray.actorc                 S   s   t | jdt t | dt d S )N_actor_method_call_modify_class)_wActorHandler   r  mrK   rK   rL   r   ?  s   zpatch.<locals>._z%ray.dashboard.modules.job.job_managerc                 S   s    t | jdt t | jdt d S )N
submit_job_monitor_job_internal)r
  
JobManagerr   r  r  rK   rK   rL   r   D  s   zray.remote_functionc                 S   s   t | jdt d S )N_remote)r
  RemoteFunctionr   r  rK   rK   rL   r   I  s   r   waitput)rQ   r>   r  ray.util.tracingr  _global_is_tracing_enabledr   after_module_importedr
  r   r   r   )r  r   rK   rK   rL   patch5  s   


r  c                   C   s   t tddsd S ttjjd ttjjjjj	d ttjjjjj	d ttj
d ttj
jd ttd ttd	 ttd
 dt_d S )Nr  Fr  r  r  r	  r  r   r  r  )rQ   r>   _uremote_functionr  	dashboardmodulesjobjob_managerr  actorr  r  rK   rK   rK   rL   unpatchR  s   



r   )o
contextlibr   	functoolsr   r   r   r   typingr   r   r>   wraptr   r
  r   r	   r   ddtrace.constantsr   $ddtrace.contrib.internal.trace_utilsr   r  r   r   r   ddtrace.internal.loggerr   ddtrace.internal.moduler   ddtrace.internal.schemar   ddtrace.internal.telemetryr   _get_configddtrace.internal.utilsr   ddtrace.internal.utils.formatsr   ddtrace.propagation.httpr   	constantsr   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   span_managerr0   r1   r2   utilsr3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r   rl   r   r   rh   RAY_COMMON_MODULE_DENYLISTrv   r   _adddictgetenvrP   rM   rR   rZ   rT   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r   rK   rK   rK   rL   <module>   s    
$3N"! 6
