o ii@s ddlZddlZddlmZddlmZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl m Z ddl mZddl mZddl mZdd l mZdd l mZddlZddlZdd lmZdd lmZdd lmZddlmZddlmZddlmZddl m!Z!ddl m"Z"ddl m#Z#ddl$m%Z%ddl&m'Z'ddl&m(Z(ddl&m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl.m0Z0ddl1m2Z2ddl3m4Z4ddl3m5Z5ddl6m7Z8dd l6m9Z9dd!l:m;Z;dd"lm?Z?dd$l>m@Z@dd%l>mAZAdd&lBmCZDdd'lBmEZFdd(lGmHZHdd)lGmIZIdd*lGmJZJdd+lGmKZKdd,lGmLZLdd-lGmMZMdd.lGmNZNdd/lGmOZOdd0lGmPZPdd1lGmQZQdd2lGmRZRdd3lGmSZSdd4lGmTZTdd5lGmUZUdd6lGmVZVdd7lGmWZWdd8lGmXZXdd9lGmYZYdd:lGmZZZdd;lGm[Z[ddlGm^Z^dd?lGm_Z_dd@lGm`Z`ddAlGmaZaddBlGmbZbddClGmcZcddDlGmdZdddElGmeZeddFlGmfZfddGlGmgZgddHlGmhZhddIlGmiZiddJlGmjZjddKlGmkZkddLlGmlZlddMlGmmZmddNlGmnZnddOlGmoZoddPlGmpZpddQlGmqZqddRlGmrZrddSlGmsZsddTlGmtZtddUlGmuZuddVlGmvZvddWlGmwZwddXlGmxZxddYlGmyZyddZlGmzZzdd[lGm{Z{dd\lGm|Z|dd]l}m~Z~dd^lmZdd_lmZdd`lmZddalmZddblmZddclmZdddlmZddelmZddflmZddglmZddhlmZddilmZddjlmZddklmZddllmZddmlmZddnlmZddolmZddplmZddqlmZddrlmZddslmZddtlmZddulmZddvlmZddwlmZddxlmZddylmZddzlmZdd{lmZdd|lmZdd}lmZdd~lmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZddlmZe-eуZiddddddddddddddddddddddddddddddddddddddddddZddhZdZdZde deddfddƄZde deddfddɄZde deddfdd˄ZGdd̈́deۃZGddτdeۃZGddфdeۃZGddӄdeۃZGddՄdeۃZeGddׄd׃Zdedededeeededffdd݄ZdedededededdeddefddZGddde4Zee_dS)N) dataclass)field)Any)Callable)Literal)Optional)Sequence)Union)cast)config)patch)APMTracingEnabledFilter)Context)Span)Tracer) ERROR_MSG) ERROR_STACK) ERROR_TYPE) SpanTypes)atexit)core)forksafe) ensure_text) get_logger)generate_128bit_trace_id) rand64bits)remoteconfig_poller)Service)ServiceStatusError) get_config)telemetry_writer)TELEMETRY_APM_PRODUCT)RLock)asbool)format_trace_id)parse_tags_str) _constants) _telemetry)AGENT_MANIFEST)ANNOTATIONS_CONTEXT_ID) DECORATOR)DEFAULT_PROJECT_NAME)DEFAULT_PROMPTS_CACHE_TTL)DEFAULT_PROMPTS_TIMEOUT) DISPATCH_ON_GUARDRAIL_SPAN_START)DISPATCH_ON_LLM_SPAN_FINISH)DISPATCH_ON_LLM_TOOL_CHOICE)$DISPATCH_ON_OPENAI_AGENT_SPAN_FINISH)DISPATCH_ON_TOOL_CALL)!DISPATCH_ON_TOOL_CALL_OUTPUT_USED)EXPERIMENT_CONFIG)EXPERIMENT_CSV_FIELD_MAX_SIZE)EXPERIMENT_DATASET_NAME_KEY)EXPERIMENT_EXPECTED_OUTPUT)EXPERIMENT_ID_KEY)EXPERIMENT_NAME_KEY)EXPERIMENT_PROJECT_ID_KEY)EXPERIMENT_PROJECT_NAME_KEY)EXPERIMENT_RUN_ID_KEY)EXPERIMENT_RUN_ITERATION_KEY)EXPERIMENTS_INPUT)EXPERIMENTS_OUTPUT)INPUT_DOCUMENTS)INPUT_MESSAGES) INPUT_PROMPT) INPUT_VALUE) INSTRUMENTATION_METHOD_ANNOTATED) INTEGRATION) LLMOBS_STRUCT)LLMOBS_TRACE_ID)MCP_TOOL_CALL_INTENT)METADATA)METRICS)ML_APP) MODEL_NAME)MODEL_PROVIDER)OUTPUT_DOCUMENTS)OUTPUT_MESSAGES) OUTPUT_VALUE) PARENT_ID_KEY)&PROMPT_TRACKING_INSTRUMENTATION_METHOD)PROPAGATED_LLMOBS_TRACE_ID_KEY)PROPAGATED_ML_APP_KEY)PROPAGATED_PARENT_ID_KEY)ROOT_PARENT_ID) SESSION_ID) SPAN_KIND) SPAN_LINKS)!SPAN_START_WHILE_DISABLED_WARNING)TAGS)TOOL_DEFINITIONS)LLMObsContextProvider)EvaluatorRunner)AsyncEvaluatorType)AsyncSummaryEvaluatorType) AsyncTaskType)BaseAsyncEvaluator)BaseAsyncSummaryEvaluator) BaseEvaluator)BaseSummaryEvaluator) ConfigType)Dataset) DatasetRecord)DatasetRecordInputType) EvaluatorType) Experiment)ExperimentResult)JSONType)Project)SummaryEvaluatorType)SyncExperiment)TaskType)"_deep_eval_async_evaluator_wrapper)_deep_eval_evaluator_wrapper) _get_base_url)_is_deep_eval_evaluator)PromptOptimization)validate_dataset)validate_dataset_split)validate_evaluators)validate_optimization_task) validate_task)validate_test_dataset) ManagedPrompt) WarmCache) PromptManager)AnnotationContext) LinkTracker)_batched)_get_llmobs_data_metastruct) _get_ml_app)_get_nearest_llmobs_ancestor)_get_parent_prompt)_get_session_id)_get_span_kind)_get_span_name)_is_evaluation_span)_validate_prompt) add_span_link)enforce_message_role)get_span_links) safe_json)LLMObsEvalMetricWriter)LLMObsEvaluationMetricEvent)LLMObsExperimentsClient)LLMObsSpanData)LLMObsSpanEvent)LLMObsSpanWriter)should_use_agentless)ExportedLLMObsSpan)Message)Prompt)PromptFallback) _ErrorField)_Meta)_MetaIO) _SpanField) Documents)Messages)extract_tool_definitions)HTTPPropagator) __version__ anthropicbedrockbotocoreopenai langchain google_adk google_genaivertexai langgraphlitellmcrewai openai_agentsmcp pydantic_aiclaude_agent_sdkrequestshttpxurllib3grpcflask starlettefastapiaiohttpasynciofutures)rrrrrrr input_datar )r output_dataexpected_output)inputsoutputsexpected_outputsevaluators_resultstaskis_asyncreturncsXt|std|rt|stdt|}|jtfddtDs*tddS)Nz!task must be a callable function.z4task must be an async function (coroutine function).c3|]}|vVqdSN.0paramparamsrJ/home/ubuntu/.local/lib/python3.10/site-packages/ddtrace/llmobs/_llmobs.py z+_validate_task_signature..z=Task function must have 'input_data' and 'config' parameters.) callable TypeErrorriscoroutinefunctioninspect signature parametersall_TASK_REQUIRED_PARAMS)rrsigrrr_validate_task_signatures r evaluatorcstf}|r ttf}t||rdSt|rdSt|s,|r$td|dtd|dt|}|jt fddt DsHtd t t dS)Nz Evaluator zE must be callable or an instance of BaseEvaluator/BaseAsyncEvaluator.z2 must be callable or an instance of BaseEvaluator.c3rrrrrrrrrz0_validate_evaluator_signature..z+Evaluator function must have parameters {}.) rdrb isinstancerurrrrrr_EVALUATOR_REQUIRED_PARAMSformattuplerrvalid_base_classesrrrr_validate_evaluator_signatures$   rcstf}|r ttf}t||rdSt|s&|rtd|dtd|dt|}|jtfddt DsBtd t t dS)NzSummary evaluator zS must be callable or an instance of BaseSummaryEvaluator/BaseAsyncSummaryEvaluator.z9 must be callable or an instance of BaseSummaryEvaluator.c3rrrrrrrrrz8_validate_summary_evaluator_signature..z3Summary evaluator function must have parameters {}.) rercrrrrrrr"_SUMMARY_EVALUATOR_REQUIRED_PARAMSrrrrrr%_validate_summary_evaluator_signatures$    rc@eZdZdZdS)LLMObsExportSpanErrorz#Error raised when exporting a span.N__name__ __module__ __qualname____doc__rrrrrrc@r)LLMObsAnnotateSpanErrorz$Error raised when annotating a span.Nrrrrrrrrc@r)LLMObsSubmitEvaluationErrorz+Error raised when submitting an evaluation.Nrrrrrr%rrc@r)#LLMObsInjectDistributedHeadersErrorz0Error raised when injecting distributed headers.Nrrrrrr+rrc@r)%LLMObsActivateDistributedHeadersErrorz1Error raised when activating distributed headers.Nrrrrrr1rrc@sneZdZUdZeedZeeed<eedZ eeed<ee dZ e e e fed<de de e fdd Zd S) LLMObsSpanaLLMObs span object. Passed to the `span_processor` function in the `enable` or `register_processor` methods. Example:: def span_processor(span: LLMObsSpan) -> Optional[LLMObsSpan]: # Modify input/output if span.get_tag("omit_span") == "1": return None if span.get_tag("no_input") == "1": span.input = [] return span )default_factoryinputoutput_tagskeyrcCs |j|S)zGet a tag from the span. :param str key: The key of the tag to get. :return: The value of the tag or None if the tag does not exist. :rtype: Optional[str] )rget)selfrrrrget_tagKs zLLMObsSpan.get_tagN)rrrrrlistrr__annotations__rdictrstrrrrrrrr7s r span_kind llmobs_input llmobs_output)valuemessagesc Cst}d}d}|tj}|dur!d}tt|ddpdddg|_|tj}|dkr6|dur6d}t||_|tj}|durPd}tt|ddpJdddg|_ |tj} |dkre| dured}t| |_ |||fS) zBuild an LLMObsSpan populated for the user span processor. Routes input/output to messages or value depending on span kind. Returns (llmobs_span, input_type, output_type). rNrF ensure_asciicontentrolellmr) rrrFVALUErrrMESSAGESrr) rrr llmobs_span input_type output_type input_valueinput_messages output_valueoutput_messagesrrr_build_llmobs_spanUs(        rspanr  llmobs_metarrc Cs|tjpt}|tjpt}tt|d|||tjp d|tjp'd |tj p0i|tj p7gt |tj p?dt|tpGd|tpMd|tpSddd }|tj} | duru|dkrutd|d tjdn| dur|dkrt|} | dur| |d d <|tj} |jtr|d kr| dur| |d <|d kr|j|d d <n|dkr|jr|jddd|d d<|d kr|j|dd <|S|dkr|jr|jddd|dd<|S)z5Build and return the full meta dict for a span event.kindrcustommessagestacktype) rrr model_namemodel_providermetadatatool_definitionsintenterrorNr ^Dropping prompt on non-LLM span kind, annotating prompts is only supported for LLM span kinds.rprompt experimentrrrrrr)rrFINPUTrOUTPUTrrrLrMlowerrIr\rINTENTrrrrrPROMPTlogwarningpoprEXPECTED_OUTPUTcontextget_baggage_itemr8rr) rr rrrrrrmeta input_prompt parent_promptrrrr_build_span_metazsL        r5c"s eZdZUdZdZeddZee d<ede Z ee d<  dde e d e eege efd dffd d Zd ed dfddZd ed dfddZd ed dfddZd ed e efddZdeded e efddZd eded e efddZd ed e efddZe   dd edede ed ede ed eef d!d"Zd ed dfd#d$Zdd%d&Z dd'd(Z!dd)d*Z"e#  +           dde ed,ed-e ed.e e$ed/e ed0e ed1e ed2e ed3e ed4e ed e eege efd5e e d6ed dfd7d8Z%edd9d:Z&d;e'ee(fd dfdd?Z*e#  dd@e+dedAe edBe e'eefd e'eeff dCdDZ,e#   ddEed2e edFe e-dGe eed e.f dHdIZ/e#     +ddEed2e edJedKe ee0dLedMed e.fdNdOZ1e#   P   +ddQedEedReedSe eedTe eedUedJed2e edMed e.fdVdWZ2e#dXed dfdYdZZ3e#   [   dd\ed]ee4e e5ge6fd^eeee5gefd_e.d`e7e8dae7e9dbe ee'ee(fgefdcee'ee'ee(ffge:fdde5d2e edGe e'eefdee-dfe ee'ee'ee(ffgefdge;eee#      ldd\ed]e?d_e.d`e7e8dJed2e edGe e'eefdde e5dae e7e9dme e-d e@fdndoZAe#      ldd\ed]eBd_e.d`e7e;e8eCfdJed2e edGe e'eefdde e5dae e7e;e9eDfdme e-d eEfdpdqZFe#     ldd\ed_e.dJed2e edGe e'eefdde e5dme e-d eEfdrdsZGe# l  t ddued]ee4e e5ge6fdvee0d`e7e;e8eCfdwe-dxedye e-dGe e'eefd eGenerate and submit an LLMObs span event to be sent to LLMObs.Nr zLError generating LLMObs span event for span %s, likely due to malformed spanTexc_info) _llmobs_span_event _get_ctx_itemrXrrRenqueuerPKeyErrorr ValueErrorr,r#)rr span_eventrrrres:     zLLMObs._submit_llmobs_spancCs"t|}|r |||S||S)zWGenerate LLMObs span event using either the meta_struct path or the legacy _store path.)r"_build_span_event_from_meta_struct _build_span_event_from_ctx_items)rr llmobs_datarrrrjs  zLLMObs._llmobs_span_eventr rrc Cs|jdur|Sd}z\z7ttttf|tji|_||}|dur,WWt |dSt |t s9t dt ||WWt |Stye}ztd|j|d}|WYd}~Wt |Sd}~wwt |w)zRun the user span processor. Returns the possibly mutated span, or None if the span should be dropped. On error, logs and returns the original span unchanged. NF=User span processor must return an LLMObsSpan or None, got %r'Error in LLMObs span processor (%r): %rT)rKr rrrrFr[rr`#record_llmobs_user_processor_calledrrrr Exceptionr,r#)rr rrr#resulterrr_apply_user_span_processors*      z!LLMObs._apply_user_span_processorcCs|tjpt}|tjpt}|tjpt}t|}|s%tdt |}|dur1t d| t ||tj p>t}|tj} | durMt d|dkrXtt|ft|||\} } } || |} | durmdS| } t|| ||| | }|tjpi}t|}||||d|}t|}t|jt|jt|jd}|jt rd|d<| t|j|t!||j"t#t$|j%|j&rd nd |||pd |||d }|tj'}|r||d <|S)N#Span kind not found in span contextxML app is required for sending LLM Observability data. Ensure this configuration is set before running your application.4Failed to extract LLMObs trace ID from span context.r Tspan_idtrace_id apm_trace_id experimentsscoper#okr) rr~ parent_idrDstart_nsdurationstatusr2metrics session_idtags span_links_ddr )(rrFMETArr'rr(rrmrrn _set_ctx_itemrK PARENT_IDrVTRACE_IDrdispatchr/rryr5rJr _llmobs_tagsrrr~r$rr0r1r8rrr int duration_nsr#CONFIG)rrrrrrrrml_apprllmobs_trace_idr rruser_processed_spanr2rrrr _dd_attrsllmobs_span_eventexperiment_configrrrrpsh       z)LLMObs._build_span_event_from_meta_structc" Cs|t}|s td|dkrtt|ft}t|jt |j t |j d}t t |dt t d}|dvrS|tdurS|tpEd|d <|tpNd |d <|tpYi}|d kr~|tdur~t|d }trq|ni}|t|d<||d <||d<d} d} |tdurd} tt|tddpdddg|_|jtrd|d<|dkr|t} | r| |d<|t} | r| |d<|t} | r| |d<|t }|dkr|durd} t!t"tt#||_|t$durd} tt|t$ddpdddg|_%|t&}|dkr|durd} t!t"tt#||_%|dkr7|t'dur7|t'p2g|dd<|dkrP|t(durP|t(pKg|dd<|t)durt|t)}|dkrht*+dnLt!t,|}||dd<n@|dkrt-|}|durt.|}|r|t/j0it/j1i}t|tr|t/j2nd}n|t)}|dur||dd<|t3dur|t3pg|d <|t4}|durt||d!<|j5rt6|7t8pd|7t9pd|7t:pdd"|d#<|j;rYd}zWz2t!tttf|t<|_=|;|}|durWWt>?|dSt|ts+t@d$tA||}WntByK}zt*5d%|j;|d&}WYd}~nd}~wwWt>?|nt>?|w|jdur~| dkrl|j|dd<n| dkr~|jd'd(d|dd<|j%dur| dkr|j%|dd<n| dkr|j%d'd(d|dd<|ds|Cd|ds|Cd|tDpi}tE|}|durtFd)|GtH||tIptJ}|tK}|durtFd*t |t|j|tL||jMt!tN|jO|j5rd#nd+||g|d, }tP|}|dur|GtQ|||d-<|R|||dd|d.<|tS} t| t"r8| r8| |d/<|tT}!|!rD|!|d0<|S)1zMBuild span event from ctx_item data (legacy path for backward compatibility).rzr r}r)rrr)r  embeddingNrrrragentragent_manifestr rFrrrrr&rrrrr documents retrievalr$r%r!r"rr#rsrtTrrr{r|r) rr~rrDrrrr2rrrrrrr )UrkrXrmrrr/rrr~r$rrrrrLrMr)rIr(rrrrCrrrr0r1r8r7r>r?rAr rrrPrrOr@rNrBr,r-rrrrFrr'r+r\rHr#rrrrrrKr[rr`rurrrvr.rJrrnrrKrQrVrGrrrrrrWrrYr4)"rrrr rr2r _dd_val metadata_ddrrrrrrrprompt_json_str prompt_dict parent_spanparent_llmobs_dataparent_llmobs_inputr4r"r#user_llmobs_spanrxrrrrrrrrrrrrqZsF                                          z'LLMObs._build_span_event_from_ctx_itemsrruse_meta_structc Cstj}i|tjp dtjpd|jpdd|td|jd}|t}|r'||d<|r-||d<d}|rI|rI| t j i} | drF| d|d<| }n| t rU| t |d<| t }t|rcd|tj<|durl|||jt} | r|d|vr|| |d<|jt} | rd |vr| |d <|jt} | rd |vr| |d <|jt} | rd |vr| |d <|jt}|rd |vr||d <|jt}|rd |vr||d <|jt}|rd|vr||d<dd|DS)Nr integrationpython)versionenvservicesourcerddtrace.versionlanguager# error_typerragas experiment_idrun_id run_iteration dataset_name project_name project_idexperiment_namecSg|] \}}d||qSz{}:{}rrkvrrr dz'LLMObs._llmobs_tags..)r rrrrrr#rrrrFr[rkrEr constantsRUNNER_IS_INTEGRATION_SPAN_TAGupdater0r1r8r<r=r6r;r:r9items)rrrrrrdd_tagsrerr_type existing_tags llmobs_tagsrrrrrrrrrrrsj                    zLLMObs._llmobs_tagscCs|jtjkrdS|jj}|durdS|t}|j$|jj D]\}}}||kr7|j |fi|ddiq!WddS1sCwYdS)N_suppress_span_kind_errorT) r]rr^ _instancer;current_trace_contextr1r)rXrWannotate)rrcurrent_contextcurrent_context_id_ context_idannotation_kwargsrrrrbfs   "zLLMObs._do_annotationscCs@|j|_|j|_|j|_dt_|jr|dSdSr)rPrecreaterQrRr6_prompt_managerr\_start_servicerrrrrUts    zLLMObs._child_after_forkcCsfz |j|jWntytdYnwz|jWdSty2tdYdSw)NzError starting LLMObs writerszError starting evaluator runner)rPstartrQrr,debugrRrrrrr|s   zLLMObs._start_servicecCs\z|j|jj|jjWnty!tdYnwz |j|jWnty<tdYnwt d|j t d|j t d|j t d|jt d|jt d|jjt d |jt d |jt t|jjt t|jjt t|jjt t|jjt t|jjt t|jj t!"|j#dS) NzError stopping evaluator runnerzError stopping LLMObs writerstrace.span_starttrace.span_finishhttp.span_inject!http.activate_distributed_headersthreading.submitthreading.executionasyncio.create_taskasyncio.execute_task)$rRstoprrPperiodicrQrr,rrreset_listenersrdrg_inject_llmobs_context._activate_llmobs_distributed_context_soft_fail_current_trace_contextrJactivate_on_asyncio_create_task_on_asyncio_execute_taskr0rVon_llm_tool_choicer2 on_tool_callr3on_tool_call_output_usedr.on_guardrail_span_startr/on_llm_span_finishr1on_openai_agent_span_finishr unregisterrUrrrr _stop_servicesJ     zLLMObs._stop_serviceTintegrations_enabledrYinstrumented_proxy_urlssiteapi_keyapp_keyrrr_tracer_autocCs|jr td|jdS|tdr#ttds#tddS|p'tj t_ |p-tj t_ |p3|j |_ |p;|j p;t |_ | pAtjt_| pGtjt_|pMtjt_|pStjt_d}t}zt|durd|ntjdt_tjrtj svd}tdtj sd}td td sd t_td ttd |r||| | d|_t}|jjj j!"|d |_|j#t$%d|jj&t$%d|jj't$%d|j(t$%d|j)t$%d|jj*dt$%d|jj+j,t$%d|jj-t$%d|jj.t$%t/|jj0j1t$%t2|jj0j3t$%t4|jj0j5t$%t6|jj0j7t$%t8|jj0j9t$%t:|jj0j;t<=|jt>t?j@d td|jtj|jtj|tjd}td|WtAB|tjtj || tjtjdStAB|tjtj || tjtjw)a Enable LLM Observability tracing. :param str ml_app: The name of your ml application. :param bool integrations_enabled: set to `true` to enable LLM integrations. :param bool agentless_enabled: set to `true` to disable sending data that requires a Datadog Agent. :param set[str] instrumented_proxy_urls: A set of instrumented proxy URLs to help detect when to emit LLM spans. :param str site: Your datadog site. :param str api_key: Your datadog api key. :param str app_key: Your datadog application key. :param str project_name: Your project name used for experiments. :param str env: Your environment name. :param str service: Your service name. :param Callable[[LLMObsSpan], Optional[LLMObsSpan]] span_processor: A function that takes an LLMObsSpan and returns an LLMObsSpan or None. If None is returned, the span will be omitted and not sent to LLMObs. z%s already enabledNDD_LLMOBS_ENABLEDz_LLMObs.enable() called when DD_LLMOBS_ENABLED is set to false or 0, not starting LLMObs service)user_defined_agentless_enabledmissing_api_keyzDD_API_KEY is required for sending LLMObs data when agentless mode is enabled. Ensure this configuration is set before running your application. missing_sitezDD_SITE is required for sending LLMObs data when agentless mode is enabled. Ensure this configuration is set before running your application.DD_REMOTE_CONFIGURATION_ENABLEDFzQRemote configuration disabled because DD_LLMOBS_AGENTLESS_ENABLED is set to true.T)r;r<rrrrr llmobs_ctxrrrz'%s enabled; instrumented_proxy_urls: %s)llmobs_enabled llmobs_ml_apprllmobs_agentless_enabledzLLMObs configurations: %s)Cr\r,rr_warn_if_litellm_was_importedrNrOr#r _dd_site _dd_api_keyr8r:r+rr_llmobs_ml_app_llmobs_instrumented_proxy_urlstimetime_nsrrLrn_remote_config_enabledrdisabler enable_agentless_client_patch_integrationsrr r;_span_aggregator dd_processorsappendrronrdrgrrrrJrrrr0rVrr2rr3rr.rr/rr1rrrTproduct_activatedr!LLMOBSr`record_llmobs_enabled)clsrrrYrrrrrrrr<rrr#r apm_filter llmobs_inforrrenables!              z LLMObs.enablecCs4dtjvrddl}t|ddstddSdSdS)Nrr_datadog_patchFa_LLMObs.enable() called after litellm was imported but before it was patched. This may cause tracing issues if you are importing patched methods like 'litellm.completion' directly. To ensure proper tracing, either run your application with ddtrace-run, call ddtrace.patch_all() before importing litellm, or enable LLMObs before importing other modules.)sysmodulesrgetattrr,r-)rrrrrUs  z$LLMObs._warn_if_litellm_was_imported task_datacCs||d<dS)z.crLrrrMrOrrrrQcrLrrrM)rGrrrrQc3|]}|vVqdSrrrMrOrrrrz1LLMObs.create_dataset_from_csv..z'Input columns not found in CSV header: c3rRrrrMrOrrrrz1Expected output columns not found in CSV header: c3rRrrrMrOrrrrz*Metadata columns not found in CSV header: ci|]}||qSrrrMrowrr z2LLMObs.create_dataset_from_csv..crSrrrMrTrrrVrWcrSrrrMrTrrrVrWr)rrr r record_id canonical_id)r4)csvfield_size_limitr5openreadliner)rnseek DictReader fieldnamesanyrrhrrSr6r7dataset_bulk_uploadrE_records)rrDrrErFrGrHr1rr4original_field_size_limitr2csvfilerrowsmissing_input_columnsmissing_output_columnsmissing_metadata_columnsr/rIr)rPrGrUrcreate_dataset_from_csvs^     &  zLLMObs.create_dataset_from_csv dataset_idcCs|jj|Sr)rrSdataset_delete)rrkrrr_delete_datasetszLLMObs._delete_datasetrDroptimization_taskdataset evaluatorssummary_evaluatorslabelization_function compute_scorer max_iterationsstopping_condition dataset_split. test_datasetcCst|t|t|t|t||t|d}|dur&|j|| d}tdid|d|d|d|d|d| p_|jd | d |d |d |d |j d| d| d| d|d|Sd | d |d |d |d |j d| d| d| d|d|S)aInitialize a PromptOptimization to iteratively improve prompts using experiments. PromptOptimization runs a baseline experiment with an initial prompt, then uses an optimization task to iteratively suggest improvements based on evaluation results. :param name: The name of the prompt optimization run. :param task: The task function to execute on the dataset. Must accept parameters ``input_data`` (dict with input data for the task) and ``config`` (configuration dictionary). Should return the task output as a JSON-serializable value. :param optimization_task: Function that calls an LLM to generate improved prompts. Must accept parameters ``system_prompt`` (str), ``user_prompt`` (str), and ``config`` (dict). Should return the new prompt (str). The system_prompt contains optimization instructions loaded from template, and user_prompt contains the current prompt with evaluation examples. :param dataset: The dataset to run experiments on, created with ``LLMObs.create_dataset()`` or ``LLMObs.pull_dataset()``. :param evaluators: A list of evaluators to measure task performance. Can be either class-based evaluators (inheriting from BaseEvaluator) or function-based evaluators that accept (input_data, output_data, expected_output) parameters. Should return a JSON-serializable value or an EvaluatorResult with the evaluation results. :param summary_evaluators: list of summary evaluators (REQUIRED). Can be either class-based evaluators (inheriting from BaseSummaryEvaluator) or function-based evaluators that accept (inputs: list, outputs: list, expected_outputs: list, evaluations: dict) and return aggregated metrics. :param labelization_function: Function to generate labels from individual experiment results (REQUIRED). Takes an individual result dict (containing "evaluations" key) and returns a string label. Used to categorize examples shown to the optimization LLM. Example: ``lambda r: "Very good" if r["evaluations"]["score"] >= 0.8 else "Bad"`` :param compute_score: Function to compute the score for each iteration (REQUIRED). Takes summary_evaluations dict from the experiment result and returns a float score. Used to determine which iteration performed best. :param config: Configuration dictionary for the optimization. Must contain: - ``prompt`` (mandatory): Initial prompt template - ``model_name`` (optional): Model to use for task execution - ``evaluation_output_format`` (optional): the output format wanted - ``runs`` (optional): The number of times to run the experiment, or, run the task for every dataset record the defined number of times. Additional config values are passed through to the task function. :param project_name: The name of the project to organize optimization runs. Defaults to the project name set in ``LLMObs.enable()``. :param tags: A dictionary of string key-value tag pairs to associate with the optimization. :param max_iterations: Maximum number of optimization iterations to run. Default is 5. :param stopping_condition: Optional function to determine when to stop optimization early. Takes summary_evaluations dict from the experiment result and returns True if optimization should stop. :param dataset_split: Controls dataset splitting. Accepts: - ``False`` (default): No splitting, use full dataset for everything. - ``True``: Split with default ratios (60/20/20 without test_dataset, 80/20 with). - ``(train, valid, test)`` tuple: Custom 3-way split ratios. Must sum to 1.0. Cannot be combined with ``test_dataset``. - ``(train, valid)`` tuple: Custom 2-way split ratios. Must sum to 1.0. Requires ``test_dataset`` for the test set. :param test_dataset: Optional name of a separate test dataset. When provided, the dataset is pulled automatically, the main dataset is split into train/valid (80/20), and the test dataset is used for the final unbiased score. Implicitly enables dataset splitting. :return: PromptOptimization object. Call ``.run()`` to execute the optimization. :raises TypeError: If task, optimization_task, evaluators, or dataset have incorrect types or signatures. :raises ValueError: If config is missing required keys. Example:: def my_task(input_data, config): prompt = config["prompt"] question = input_data["question"] # Use prompt and question to generate answer return {"answer": generate_answer(prompt, question)} def optimization_task(system_prompt, user_prompt, config): # Call LLM to analyze results and generate improved prompt import openai client = openai.OpenAI(api_key="your-api-key") response = client.chat.completions.create( model="gpt-4", messages=[ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ], response_format={"type": "json_object"}, ) new_prompt = json.loads(response.choices[0].message.content)["new_prompt"] return new_prompt def accuracy_evaluator(input_data, output_data, expected_output): is_correct = output_data["answer"] == expected_output["answer"] return "correct" if is_correct else "incorrect" def summary_evaluator(inputs, outputs, expected_outputs, evaluations): correct_count = sum(1 for e in evaluations if e == "correct") return {"accuracy": correct_count / len(evaluations)} def compute_score(summary_evaluations): return summary_evaluations["summary_evaluator"]["value"]["accuracy"] def labelization_function(individual_result): eval_value = individual_result["evaluations"]["accuracy_evaluator"]["value"] return "Correct answer" if eval_value == "correct" else "Incorrect answer" dataset = LLMObs.create_dataset(name="qa_pairs", ...) opt = LLMObs._prompt_optimization( name="optimize_qa", task=my_task, optimization_task=optimization_task, dataset=dataset, evaluators=[accuracy_evaluator], labelization_function=labelization_function, compute_score=compute_score, summary_evaluators=[summary_evaluator], config={ "prompt": "Answer the question: {question}", "model_name": "gpt-3.5-turbo", "runs": 10 } ) results = opt.run() N)rrrDrrorprqrr rrrtrs_llmobs_instancerrurvrwrxr) r{rzrwr|rxryr0rvr:r)rrDrrorprqrrrsrtr rrrurvrwrxpulled_test_datasetrrr_prompt_optimizations             zLLMObs._prompt_optimizationrunsc  Cst|ddt|tstd|stdt|} t| D]\} } t| ddt| r2t| | | <qq| rBt dd| DsBtd| rO| D]}t |ddqFt |||| |pX|j ||||j | | d S) a Initializes an Experiment to run a task on a Dataset and evaluators. :param name: The name of the experiment. :param task: The task function to run. Must accept parameters ``input_data`` and ``config``. :param dataset: The dataset to run the experiment on, created with LLMObs.pull/create_dataset(). :param evaluators: A list of evaluator functions or BaseEvaluator instances to evaluate the task output. Function-based evaluators must accept parameters ``input_data``, ``output_data``, and ``expected_output``. Class-based evaluators must inherit from BaseEvaluator and implement the evaluate method. :param project_name: The name of the project to save the experiment to. :param description: A description of the experiment. :param tags: A dictionary of string key-value tag pairs to associate with the experiment. :param config: A configuration dictionary describing the experiment. :param summary_evaluators: A list of summary evaluator functions or BaseSummaryEvaluator instances to evaluate the task results and evaluations to produce a single value. Function-based summary evaluators must accept parameters ``inputs``, ``outputs``, ``expected_outputs``, ``evaluators_results``. Class-based summary evaluators must inherit from BaseSummaryEvaluator and implement the evaluate method which receives a SummaryEvaluatorContext. :param runs: The number of times to run the experiment, or, run the task for every dataset record the defined number of times. Fr)Dataset must be an LLMObs Dataset object.zKEvaluators must be a list of callable functions or BaseEvaluator instances.css"|] }t|p t|tVqdSr)rrre)rsummary_evaluatorrrrrs  z$LLMObs.experiment..zZSummary evaluators must be a list of callable functions or BaseSummaryEvaluator instances.rrr1r ryrrr})rrrgrr enumeraterrursrrrpr:rrrDrrprqr1rrr rrr}evaluators_listidxrrrrrr&sD $    zLLMObs.experimentc  Cst|ddt|tstd|stdt|} t| D]\} } t| ddt| r2t| | | <qq| r@| D]}t |ddq7t |||| |pI|j ||||j | | d S)aEInitializes an Experiment to run an async task on a Dataset with evaluators. This is the async version of experiment() that supports async tasks, evaluators, and summary evaluators. Sync evaluators are also supported and will be run via asyncio.to_thread(). :param name: The name of the experiment. :param task: The async task function to run. Must be an async function accepting parameters ``input_data`` and ``config``. :param dataset: The dataset to run the experiment on, created with LLMObs.pull/create_dataset(). :param evaluators: A list of evaluator functions or BaseEvaluator/BaseAsyncEvaluator instances. Supports both sync and async evaluators. Sync evaluators will be run in a thread pool. Function-based evaluators must accept parameters ``input_data``, ``output_data``, and ``expected_output``. Class-based evaluators must inherit from BaseEvaluator or BaseAsyncEvaluator and implement the evaluate method. :param project_name: The name of the project to save the experiment to. :param description: A description of the experiment. :param tags: A dictionary of string key-value tag pairs to associate with the experiment. :param config: A configuration dictionary describing the experiment. :param summary_evaluators: A list of summary evaluator functions or BaseSummaryEvaluator/ BaseAsyncSummaryEvaluator instances. Supports both sync and async. Function-based summary evaluators must accept parameters ``inputs``, ``outputs``, ``expected_outputs``, ``evaluators_results``. Class-based summary evaluators must inherit from BaseSummaryEvaluator or BaseAsyncSummaryEvaluator and implement the evaluate method. :param runs: The number of times to run the experiment. Tr~rz`Evaluators must be a list of callable functions, BaseEvaluator, or BaseAsyncEvaluator instances.r) rrrgrrrrrurrrrkr:rrrrrasync_experiment s< )   zLLMObs.async_experimentc Cs:t|tj|g|p |j||||j|dd }|jddd|S)NT)rrr1r ryr}is_distributedr F) ensure_unique)rk _NO_OP_TASKr:r_setup_experiment) rrDrpr1rrr r}r&rrr_distributed_experimentQs$  zLLMObs._distributed_experimentrrdataset_recordsjobs raise_errorsrc Cs|jr|jjs td|jj|} |j| _|| j_|| _|| _ | |||} zt Wnt y>t | } Y| | fSwddl} | jjdd} | t j| } Wd| | fS1sawY| | fS)Nr rr|) max_workers)rr\rnrSexperiment_getry_datasetrc_task _evaluators_run_task_single_iterationrget_running_loop RuntimeErrorrunconcurrent.futuresrThreadPoolExecutorsubmitrw)rrrrrqrrrrr&cororesults concurrentpoolrrr_run_for_experimentos*     zLLMObs._run_for_experiment processorcCs ||j_dS)aRegister a processor to be called on each LLMObs span. This can be used to modify the span before it is sent to LLMObs. For example, you can modify the input/output. You can also return None to omit the span entirely from being sent to LLMObs. To deregister the processor, call `register_processor(None)`. :param processor: A function that takes an LLMObsSpan and returns an LLMObsSpan or None. If None is returned, the span will be omitted and not sent to LLMObs. N)rrK)rrrrrregister_processors zLLMObs.register_processorrcCs|tvrdSt|tjvSNF)SUPPORTED_LLMOBS_INTEGRATIONSrI_monkey_get_patched_modules)rrrrr_integration_is_enabledszLLMObs._integration_is_enabledcCsh|js td|jdStd|jt|j|jd|_d|_ t t j dtd|jdS)Nz%s not enabledz Disabling %sFz %s disabled)r\r,rrrrr rrrr rr!r)rrrrr s  zLLMObs.disablecs@|sdSfdd|D}t}|r||}t|dS)Ncs<g|]}|dtdkr|dtdkr|qSr~r)r6 export_span)r span_linkrrrrs z*LLMObs._tag_span_links..)rkrYr)rrrcurrent_span_linksrrr_tag_span_linkss  zLLMObs._tag_span_linksr% _linked_spanscsPtddifddfdd}fdd}t||S) a Sets specified attributes on all LLMObs spans created while the returned AnnotationContext is active. Annotations are applied in the order in which annotation contexts are entered. :param tags: Dictionary of JSON serializable key-value tag pairs to set or update on the LLMObs span regarding the span's context. :param prompt: A dictionary that represents the prompt used for an LLM call in the following form: `{ "id": "...", "version": "...", "chat_template": [{"content": "...", "role": "..."}, ...], "variables": {"variable_1": "...", ...}}`. "tags": {"key1": "value1", "key2": "value2"}, }` Can also be set using the `ddtrace.llmobs.utils.Prompt` constructor class. For managed prompts, use `prompt.to_annotation_dict(**variables)`. - This argument is only applicable to LLM spans. - The dictionary may contain optional keys relevant to Templates and RAG applications: `rag_context_variables` - a list of variable key names that contain ground truth context information `rag_query_variables` - a list of variable key names that contains query information for an LLM call :param name: set to override the span name for any spans annotated within the returned context. created_contextNcsxjj}}|dur(tdd}|t|d|_jjj||d<|S| ts5|t||S| t}|S)NF) is_remoteTr) rr;rrset_baggage_itemr) _reactivatecontext_providerrr1) current_ctxctx_id annotation_idrstaterrget_annotations_context_ids     z=LLMObs.annotation_context..get_annotations_context_idc sRjj}jj|dfWddS1s"wYdS)N)rr%_namer)rrXrWr)r)rrrrrDr%rrrregister_annotations "z6LLMObs.annotation_context..register_annotationcsjj(tjjD]\}\}}}|krjj|nq tdWdn1s/wYddurVdd_jjj }|durXjjj ddSdSdS)Nz Failed to pop annotation contextrF) rrXrrWr.r,rrr;ractiver)irrcurrent_activerrrderegister_annotations    z8LLMObs.annotation_context..deregister_annotation)rr)rrr%rDrrrr)rrrrrDr%rrrannotation_contexts ! zLLMObs.annotation_contextrcCsJ|j|j}|dur|}||_|WdS1swYdS)z5Thread-safe get-or-initialize for the prompt manager.N)_prompt_manager_lockr_initialize_prompt_manager)rmanagerrrr_ensure_prompt_manager s$zLLMObs._ensure_prompt_manager prompt_idlabel) development productionfallbackcCs|}||||S)a Retrieve a prompt template from the Datadog Prompt Registry. :param prompt_id: The unique identifier of the prompt in the registry :param label: Deployment label (e.g., "production", "development"). If not provided, returns the latest version. :param fallback: Fallback to use if prompt cannot be fetched (cold start + API failure). Can be a template string, message list, Prompt dict, or a callable that returns any of those. :returns: A ManagedPrompt object with template and rendering methods :raises ValueError: If the prompt cannot be fetched and no fallback is provided Example:: # Simple usage - returns the latest version prompt = LLMObs.get_prompt("greeting") messages = prompt.format(user="Alice") # With explicit label and fallback prompt = LLMObs.get_prompt( "greeting", label="production", fallback="Hello {{user}}, how can I help?" ) # Use with annotation_context for observability # Pass the same variables to both format() and to_annotation_dict() prompt = LLMObs.get_prompt("greeting") variables = {"user": "Alice"} with LLMObs.annotation_context(prompt=prompt.to_annotation_dict(**variables)): openai.chat.completions.create( messages=prompt.format(**variables) ) )r get_prompt)rrrrprompt_managerrrrr*s)zLLMObs.get_prompthotwarmcCsD|jdur|jj||ddS|r td}t|d}|dSdS)zClear the prompt cache. Args: hot: If True, clear the hot (in-memory) cache. Defaults to True. warm: If True, clear the warm (file-based) cache. Defaults to True. N)rrDD_LLMOBS_PROMPTS_CACHE_DIR) cache_dir)r clear_cache _get_configr~clear)rrrr warm_cacherrrclear_prompt_cacheVs   zLLMObs.clear_prompt_cachecCs|}|||S)aZForce refresh a specific prompt from the registry. Fetches the prompt synchronously and updates both caches. Args: prompt_id: The prompt identifier. label: The prompt label. If not provided, returns the latest version. Returns: The refreshed prompt, or None if fetch failed. )rrefresh_prompt)rrrrrrrrfs zLLMObs.refresh_promptcCsftj}|s tdtdtt}tddt}td}tdtt}tdp(dtj}t ||||||d S) z1Initialize the prompt manager with configuration.z.DD_API_KEY is required for the Prompt RegistryDD_LLMOBS_PROMPTS_CACHE_TTL$DD_LLMOBS_PROMPTS_FILE_CACHE_ENABLEDFrDD_LLMOBS_PROMPTS_TIMEOUTDD_LLMOBS_OVERRIDE_ORIGINz https://api.)rr+ cache_ttlfile_cache_enabledrr@) r rrnrr,rMr#r-rr)rrrrrr@r+rrrrzs    z!LLMObs._initialize_prompt_managercCs|jdur tddSd}z|jjWnty(d}tjdddYnwz|jj|jjWntyId}tjd ddYnwt |dS) z[ Flushes any remaining spans and evaluation metrics to the LLMObs backend. FzNflushing when LLMObs is disabled. No spans or evaluation metrics will be sent.Nevaluator_flush_errorzFailed to run evaluator runner.Trhwriter_flush_errorz4Failed to flush LLMObs spans and evaluation metrics.) r\r,r-rrRrrvrPrQr`record_user_flush)rr#rrrflushs$     z LLMObs.flushcCsddtD}|D]\}}d|}|tjvr%ttj|||<q td}t|}| dd|Dt d ddi|dd |D}t d |d S) z~ Patch LLM integrations. Ensure that we do not ignore DD_TRACE__ENABLED or DD_PATCH_MODULES settings. cSs"i|] }||dkr ddgndqS)rzbedrock-runtimezbedrock-agent-runtimeTr)rrrrrrVsz.LLMObs._patch_integrations..zDD_TRACE_%s_ENABLEDDD_PATCH_MODULEScSs&i|]\}}|tvr|t|qSr)rvaluesr#rrrrrVs&rTcSsg|]\}}|r|qSrrrrrrrrQz.LLMObs._patch_integrations..zPatched LLM integrations: %sNr) rrrupperrNenvironr#rOr%rr r,r)integrations_to_patchmodulerenv_vardd_patch_modulesdd_patch_modules_to_strllm_patched_modulesrrrr s    zLLMObs._patch_integrationsc Cs|js tddS|dur!|j}|dur!t|dtdd}z6z%|jt j kr1d}tdt t |j t|tp>|jdWWt||SttfyYd}tddwt||w) zReturns a simple representation of a span to export its span and trace IDs. If no span is provided, the current active LLMObs-type span will be used. zNLLMObs.export_span() called when LLMObs is disabled. No span will be exported.Nno_active_spanzNo span provided and no active LLMObs-generated span found. Ensure you pass the span explicitly using LLMObs.export_span(span=) when exporting from a different thread or async task than where the span was created. invalid_span&Span must be an LLMObs-generated span.rz8Failed to export span. Span must be a valid Span object.)r\r,r-r _current_spanr`record_span_exportedrr]rr^rrr~r$rkrGrrAttributeError)rrr#rrrrs2     zLLMObs.export_spancCs|j}t|tr |SdS)zReturns the currently active LLMObs-generated span. Note that there may be an active span represented by a context object (i.e. a distributed trace) which will not be returned by this method. N)rJrrr)rrrrrrs zLLMObs._current_spancCsN|j}t|tr |St|tr%|j}t|tpt|j |j t <|SdS)z1Returns the context for the current LLMObs trace.N) rJrrrrr0rrkrGr_metarS)rrr0rrrrs    zLLMObs._current_trace_contextcCs|j}|r2|tt|jt|tr|t n|j t }|dur(t |n|j}|t |n |tt|t t|j|dS)z\Propagate the llmobs parent span's ID as the new span's parent ID and activate the new span.N)rJrrrQrr~rrrkrGrrrSrrrVrr)rr llmobs_parentparent_llmobs_trace_idrrrrr_s   zLLMObs._activate_llmobs_spanoperation_kindrr _decoratorc Cs|dur|}|jj||tjd}|js|S|t||dur%|t||dur/|t||dur5|nt |}|durC|t ||durI|nt |}|durUt d| t|t|t|itd||||S)N)resourcer]zml_app is required for sending LLM Observability data. Ensure the name of your LLM application is set via `DD_LLMOBS_ML_APP` or `LLMObs.enable(ml_app='...')`before running your application.z3Starting LLMObs span: %s, span_kind: %s, ml_app: %s)r;tracerr^r\rrXrLrMrrWrrn_set_ctx_itemsr*rKr,r) rrrDrrrrrrrrr _start_spans4     zLLMObs._start_spanc CF|jdur tt|durd}|durd}|jjd||||||dS)a Trace an invocation call to an LLM where inputs and outputs are represented as text. :param str model_name: The name of the invoked LLM. If not provided, a default value of "custom" will be set. :param str name: The name of the traced operation. If not provided, a default value of "llm" will be set. :param str model_provider: The name of the invoked LLM provider (ex: openai, bedrock). If not provided, a default value of "custom" will be set. :param str session_id: The ID of the underlying user session. Required for tracking sessions. :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default value will be set to the value of `DD_LLMOBS_ML_APP`. :returns: The Span object representing the traced operation. FNrr rrrrrr\r,r-rZrrrrrDrrrrrrrr 2s  z LLMObs.llmcC*|jdur tt|jjd||||dS)a( Trace a call to an external interface or API. :param str name: The name of the traced operation. If not provided, a default value of "tool" will be set. :param str session_id: The ID of the underlying user session. Required for tracking sessions. :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default value will be set to the value of `DD_LLMOBS_ML_APP`. :returns: The Span object representing the traced operation. FtoolrDrrrrrrDrrrrrrrY  z LLMObs.toolcCr)aK Trace a standalone non-LLM operation which does not involve an external request. :param str name: The name of the traced operation. If not provided, a default value of "task" will be set. :param str session_id: The ID of the underlying user session. Required for tracking sessions. :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default value will be set to the value of `DD_LLMOBS_ML_APP`. :returns: The Span object representing the traced operation. Frrrrrrrrurz LLMObs.taskcCr)aj Trace a dynamic workflow in which an embedded language model (agent) decides what sequence of actions to take. :param str name: The name of the traced operation. If not provided, a default value of "agent" will be set. :param str session_id: The ID of the underlying user session. Required for tracking sessions. :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default value will be set to the value of `DD_LLMOBS_ML_APP`. :returns: The Span object representing the traced operation. Frrrrrrrrrz LLMObs.agentcCr)a3 Trace a predefined or static sequence of operations. :param str name: The name of the traced operation. If not provided, a default value of "workflow" will be set. :param str session_id: The ID of the underlying user session. Required for tracking sessions. :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default value will be set to the value of `DD_LLMOBS_ML_APP`. :returns: The Span object representing the traced operation. FworkflowrrrrrrrrzLLMObs.workflowc Cr)a Trace a call to an embedding model or function to create an embedding. :param str model_name: The name of the invoked embedding model. If not provided, a default value of "custom" will be set. :param str name: The name of the traced operation. If not provided, a default value of "embedding" will be set. :param str model_provider: The name of the invoked LLM provider (ex: openai, bedrock). If not provided, a default value of "custom" will be set. :param str session_id: The ID of the underlying user session. Required for tracking sessions. :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default value will be set to the value of `DD_LLMOBS_ML_APP`. :returns: The Span object representing the traced operation. FNrrrrrrrrrs  zLLMObs.embeddingcCr)al Trace a vector search operation involving a list of documents being returned from an external knowledge base. :param str name: The name of the traced operation. If not provided, a default value of "workflow" will be set. :param str session_id: The ID of the underlying user session. Required for tracking sessions. :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default value will be set to the value of `DD_LLMOBS_ML_APP`. :returns: The Span object representing the traced operation. FrrrrrrrrrzLLMObs.retrievalrrrc Cs|jdur tt|jjd|||d} |r| jt||r&| jt ||dur1| jt ||r:| jt || rC| jt | |rL| jt || rU| jt| | S)a Trace an LLM experiment, only used internally by the experiments SDK. :param str name: The name of the traced operation. If not provided, a default value of "agent" will be set. :param str session_id: The ID of the underlying user session. Required for tracking sessions. :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default value will be set to the value of `DD_LLMOBS_ML_APP`. :param str experiment_id: The ID of the experiment to associate with this span and its children. :returns: The Span object representing the traced operation. Fr&)rDrrN)r\r,r-rZrrr0rr8r<r=r6r:r;r9) rrDrrrrrrrrrrrrr _experiment s$  zLLMObs._experimentrrr rr!rrc  Csd} z[|dur|j}|durd} td|jtjkr#d} td|jr,d} td|durBt|t s;d} td | |t ||durct|t rVt d d | Ds\d } td | |t||durt|t srd} td|d} | r|tt| | |t||durt|}|r|t||t}| dur| |_|durzt|dd}| |t|| |tttiWnttfy}z d} tdt|d}~ww|s| stdd}|dus|dur0|dkr|j|||d\}} n6|dkr |j |||d\}} n&|dkr|j!|||d\}} n|dkr(|j"|||dn|j#|||d| rOt| t$rO| D]}t%||dd|d dd!d"q;|rVt|Wt&'|| dSt&'|| w)#aT Sets metadata, inputs, outputs, tags, and metrics as provided for a given LLMObs span. Note that with the exception of tags, this method will override any existing values for the provided fields. :param Span span: Span to annotate. If no span is provided, the current active span will be used. Must be an LLMObs-type span, i.e. generated by the LLMObs SDK. :param prompt: A dictionary that represents the prompt used for an LLM call in the following form: `{ "id": "...", "template": "...", "chat_template": [{"content": "...", "role": "..."}, ...]) "version": "...", "variables": {"variable_1": "...", ...}, tags": {"tag_1": "...", ...}, }`. Can also be set using the `ddtrace.llmobs.utils.Prompt` constructor class. - This argument is only applicable to LLM spans. - The dictionary may contain two optional keys relevant to RAG applications: `rag_context_variables` - a list of variable key names that contain ground truth context information `rag_query_variables` - a list of variable key names that contains query information for an LLM call :param input_data: A single input string, dictionary, or a list of dictionaries based on the span kind: - llm spans: accepts a string, or a dictionary of form {"content": "...", "role": "...", "tool_calls": ..., "tool_results": ...}, where "tool_calls" are an optional list of tool call dictionaries with required keys: "name", "arguments", and optional keys: "tool_id", "type", and "tool_results" are an optional list of tool result dictionaries with required key: "result", and optional keys: "name", "tool_id", "type" for function calling scenarios. - embedding spans: accepts a string, list of strings, or a dictionary of form {"text": "...", ...} or a list of dictionaries with the same signature. - other: any JSON serializable type. :param output_data: A single output string, dictionary, or a list of dictionaries based on the span kind: - llm spans: accepts a string, or a dictionary of form {"content": "...", "role": "...", "tool_calls": ...}, where "tool_calls" are an optional list of tool call dictionaries with required keys: "name", "arguments", and optional keys: "tool_id", "type" for function calling scenarios. - retrieval spans: a dictionary containing any of the key value pairs {"name": str, "id": str, "text": str, "score": float}, or a list of dictionaries with the same signature. - other: any JSON serializable type. :param metadata: Dictionary of JSON serializable key-value metadata pairs relevant to the input/output operation described by the LLMObs span. :param tags: Dictionary of JSON serializable key-value tag pairs to set or update on the LLMObs span regarding the span's context. :param tool_definitions: list of tool definition dictionaries for tool calling scenarios. - This argument is only applicable to LLM spans. - Each tool definition is a dictionary containing a required "name" (string), and optional "description" (string) and "schema" (JSON serializable dictionary) keys. :param metrics: Dictionary of JSON serializable key-value metric pairs, such as `{prompt,completion,total}_tokens`. Ninvalid_span_no_active_spanszNo span provided and no active LLMObs-generated span found. Ensure you pass the span explicitly using LLMObs.annotate(span=, ...) when annotating from a different thread or async task than where the span was created.invalid_span_typerinvalid_finished_spanz Cannot annotate a finished span.invalid_metadatazmetadata must be a dictionarycss|] }t|ttfVqdSr)rrrM)rrrrrr sz"LLMObs.annotate..invalid_metricszAmetrics must be a dictionary of string key - numeric value pairs. invalid_tagszEspan tags must be a dictionary of string key - primitive value pairs.rF)strict_validationinvalid_promptz%Failed to validate prompt with error:zBSpan kind not specified, skipping annotation for input/output datar )rrr)input_documents output_textr) input_textoutput_documentsr&)rrr~rrrr)(rrrr]rr^finishedrrr_set_dict_attributerIrrrJrrrWrr[rr\rkrXrDrrBrRrDrnr _tag_llm_io_tag_embedding_io_tag_retrieval_io_tag_freeform_io _tag_text_iorrr`record_llmobs_annotate)rrr%rrr rrr!rrrr#rvalidated_tool_definitionsrvalidated_promptrxannotation_error_message linked_spanrrrr@ sC              zLLMObs.annotatecCs|dur$zt|tst|}|jr|t|jWn ty#YdSw|dur*dSzt|ts4t|}|js:WdS|t|jWdStyMYdSw)zTags input/output messages for LLM-kind spans. Will be mapped to span's `meta.{input,output}.messages` fields. N)zFailed to parse input messages.invalid_io_messagesNN)z Failed to parse output messages.r)rrrrrArrO)rrrrrrrr s,    zLLMObs._tag_llm_iocCsh|dur$zt|tst|}|jr|t|jWn ty#YdSw|dur*dS|tt|dS)zTags input documents and output text for embedding-kind spans. Will be mapped to span's `meta.{input,output}.text` fields. N)z Failed to parse input documents.invalid_embedding_ior)rrrrr@rrPr)rrr r rrrr s  zLLMObs._tag_embedding_iocCsl|dur |tt||durdSzt|tst|}|js"WdS|t|jWdSty5YdSw)zTags input text and output documents for retrieval-kind spans. Will be mapped to span's `meta.{input,output}.text` fields. Nr)z!Failed to parse output documents.invalid_retrieval_io)rrCrrrrrNr)rrr rrrrr s  zLLMObs._tag_retrieval_iocCs8|dur |tt||dur|tt|dSdS)zTags input/output values for non-LLM kind spans. Will be mapped to span's `meta.{input,output}.values` fields. N)rrCrrPrrrrrrrr" s zLLMObs._tag_text_iocCs0|dur |t||dur|t|dSdS)a Tags input/output values for experient spans. Will be mapped to span's `meta.{input,output}` fields. this is meant to be non restrictive on user's data, experiments allow arbitrary structured or non structured IO values in its spans N)rr>r?rrrrr, s  zLLMObs._tag_freeform_iorcCs(||pi}|||||dS)zSets a given LLM Obs span attribute with a dictionary key/values. If the attribute is already set on the span, the new dict with be merged with the existing dict. N)rkrr)rrrexisting_valuerrrr8 s zLLMObs._set_dict_attribute metric_typespan_with_tag_value timestamp_ms assessment reasoningc Cs||jdur tdddSd} i} z|du|duA}|s#d} td|durGt|tr<| rt| tsd?} td@t| } | rt| trt| |dA<|jj|Wt| || dSt| || w)Ba Submits a custom evaluation metric for a given span. :param str label: The name of the evaluation metric. :param str metric_type: The type of the evaluation metric. One of "categorical", "score", "boolean". :param value: The value of the evaluation metric. Must be a string (categorical), integer (score), float (score), or boolean (boolean). :param dict span: A dictionary of shape {'span_id': str, 'trace_id': str} uniquely identifying the span associated with this evaluation. :param dict span_with_tag_value: A dictionary with the format {'tag_key': str, 'tag_value': str} uniquely identifying the span associated with this evaluation. :param tags: A dictionary of string key-value pairs to tag the evaluation metric with. :param str ml_app: The name of the ML application :param int timestamp_ms: The unix timestamp in milliseconds when the evaluation metric result was generated. If not set, the current time will be used. :param dict metadata: A JSON serializable dictionary of key-value metadata pairs relevant to the evaluation metric. :param str assessment: An assessment of this evaluation. Must be either "pass" or "fail". :param str reasoning: An explanation of the evaluation result. Fz>LLMObs.submit_evaluation() called when LLMObs is not enabled. z(Evaluation metric data will not be sent.N&provided_both_span_and_tag_joining_keyz`Exactly one of `span` or `span_with_tag_value` must be specified to submit an evaluation metric.r~rrz`span` must be a dictionary containing both span_id and trace_id keys. LLMObs.export_span() can be used to generate this dictionary from a given span.rtag_key tag_valueinvalid_joining_keyza`span_with_tag_value` must be a dict with keys 'tag_key' and 'tag_value' containing string values)rrtagirinvalid_timestampzTtimestamp_ms must be a non-negative integer. Evaluation metric data will not be sentinvalid_metric_labelz:label must be the specified name of the evaluation metric..invalid_label_valuez#label value must not contain a '.'.) categoricalscorebooleanjsoninvalid_metric_typezHmetric_type must be one of 'categorical', 'score', 'boolean', or 'json'.r/invalid_metric_valuez0value must be a string for a categorical metric.r0z5value must be an integer or float for a score metric.r1z-value must be a boolean for a boolean metric.r2z'value must be a dict for a json metric.z4tags must be a dictionary of string key-value pairs.missing_ml_appzML App name is required for sending evaluation metrics. Evaluation metric data will not be sent. Ensure this configuration is set before running your application.)rrrzBFailed to parse tags. Tags for evaluation metrics must be strings.otelrjoin_onrr!r#z{}_valuerrcSrrrrrrrr rz,LLMObs.submit_evaluation..)passfailinvalid_assessmentzGFailed to parse assessment. assessment must be either 'pass' or 'fail'.r$invalid_reasoningz6Failed to parse reasoning. reasoning must be a string.r%rz.metadata must be json serializable dictionary.r )r\r,rrnrrrrrrrr)rMboolrr rrrr_otel_trace_enabledrrr2loadsrrQrlr`record_llmobs_submit_evaluation)rrr!rrr"rrr#r r$r%r#r7has_exactly_one_joining_keyevaluation_tagsrrevaluation_metricrrrsubmit_evaluationB s #      "zLLMObs.submit_evaluation span_contextrequest_headerscCs|jdurdS|jj}t|tr|jn|}|dur t|jnt }d}t|tr4| t }| t }n|durK|j tp@tj}|j tpIt}ntj}t}||j t<t||j t<|durh||j t<dSdSr)r\rrJrrrr0rr~rVrkrKrGrrrTr rrSrrU)rrDrE active_spanactive_contextrrrrrrr s&      zLLMObs._inject_llmobs_contextc Cs|jdur td|Sd}z7t|tsd}td|dur$|jj}|dur.d}tdt|t s7tdt |j ||Wt |St |w) zFInjects the span's distributed context into the given request headers.FzpLLMObs.inject_distributed_headers() called when LLMObs is not enabled. Distributed context will not be injected.Ninvalid_request_headersz?request_headers must be a dictionary of string key-value pairs.rz4No span provided and no currently active span found.zKspan must be a valid Span object. Distributed context will not be injected.)r\r,r-rrrrr; current_spanrrinjectr0r`!record_inject_distributed_headers)rrErr#rrrinject_distributed_headers s0    z!LLMObs.inject_distributed_headersr0cCs|j||dddS)NT _soft_fail)$_activate_llmobs_distributed_contextrrEr0rrrr4 sz5LLMObs._activate_llmobs_distributed_context_soft_failrNc Cshd}z|jdurWt|dS|jr|js+d}|r'tdWt|dStd|j t }|durDd}t dWt|dSzt |}Wnt yad}tdYWt|dSw|j t}|durt dt|j|d }t|j|jt<|jj|d }Wt|dSt|j|d }t||jt<|jj|Wt|dSt|w) NFmissing_contextz5Failed to extract trace/span ID from request headers.missing_parent_idz8Failed to extract LLMObs parent ID from request headers.invalid_parent_idz6Failed to parse LLMObs parent ID from request headers.zFailed to extract LLMObs trace ID from request headers. Expected string, got None. Defaulting to the corresponding APM trace ID.)rr~missing_parent_llmobs_trace_id)r\r`#record_activate_distributed_headersrr~r,r-rrrrUrrrnrSrrrrJr) rrEr0rNr# _parent_idrrllmobs_contextrrrrO8 sR !        z+LLMObs._activate_llmobs_distributed_contextcCsH|jdur tddSt|}|jjj||jj ||dddS)z Activates distributed tracing headers for the current request. :param request_headers: A dictionary containing the headers for the current request. FzsLLMObs.activate_distributed_headers() called when LLMObs is not enabled. Distributed context will not be activated.NrM) r\r,r-rextractrr;rrrOrPrrractivate_distributed_headersb s  z#LLMObs.activate_distributed_headersr)NFN)rN) NTNNNNNNNNNNF)NNN)NrNFT)NNrCrNT)NNrnNFN)rNNNNr|)rNNNr|)r|FrNr)NNNN)TT)NNNNNF)NNNF) NNNNNNNNNN) NNNNNNNNNNF)NNNNNNNN)F)}rrrrr\rNrOr8rrr+r:rrrrrHrrdrgrerrjrryrprq staticmethodr<rrrbrUrr classmethodsetrrrrrrrdr-rrgr0rhrBrjrmrirfrmrrjrorMr rrvr{rqrpr&rar_r`rkrrrlrrrr rrrrrrrr"rrrrr}rrrrrr rrrrr_rr rrrrrrrrrrrrrrobjectrCrrLrrOrY __classcell__rrrZrr6s2 &CD H   ,       !        ;     G        '    F    F   (    \   +      * &     '     2     $"""         9 .   )$r6)rrZ dataclassesrrrr2r9rNrrtypingrrrrrr r urllib.parser&rIr r ddtrace._trace.apm_filterr ddtrace._trace.contextrddtrace._trace.spanrddtrace._trace.tracerrddtrace.constantsrrr ddtrace.extrddtrace.internalrrrddtrace.internal.compatrddtrace.internal.loggerrddtrace.internal.nativerr$ddtrace.internal.remoteconfig.workerrddtrace.internal.servicerrddtrace.internal.telemetryrrr $ddtrace.internal.telemetry.constantsr!ddtrace.internal.threadsr"ddtrace.internal.utils.formatsr#r$r%ddtrace.llmobsr&rr'r`ddtrace.llmobs._constantsr(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\ddtrace.llmobs._contextr]!ddtrace.llmobs._evaluators.runnerr^ddtrace.llmobs._experimentr_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrsrtru#ddtrace.llmobs._prompt_optimizationrvrwrxryrzr{r|ddtrace.llmobs._promptsr}ddtrace.llmobs._prompts.cacher~ddtrace.llmobs._prompts.managerrddtrace.llmobs._utilsrrrrrrrrrrrrrrrrddtrace.llmobs._writerrrrrrrrddtrace.llmobs.typesrrrrrrrrddtrace.llmobs.utilsrrrddtrace.propagation.httprddtrace.versionrrr,rrrrr<rrrrvrrrrrrrrrr5r6rrrrrs$                                                                                                                                                                         % 4 Z