o
    iz#                     @   s   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZ d dl
mZ d d	lmZ d d
lmZ d dlmZ eeZG dd dZdedefddZG dd dZdS )    N)Optional)Union)config)
get_logger)telemetry_writer)TELEMETRY_LOG_LEVEL)TELEMETRY_NAMESPACE)parse_version)INTERNAL_CONTEXT_VARIABLE_KEYS)INTERNAL_QUERY_VARIABLE_KEYSc                   @   s   e Zd ZdZdddZdS )RagasDependenciesz
    A helper class to store instances of ragas classes and functions
    that may or may not exist in a user's environment.
    returnNc                 C   s  dd l }|j| _t|j}|dks|dk rtd| jddlm} || _ddlm	} || _	ddl
m} || _ddlm} || _dd	l
m} || _dd
lm} || _ddl
m}	 |	| _ddlm}
 |
| _ddlm} || _ddlm} || _ddlm} || _ddlm} || _d S )Nr   )r      r   )r      
   z"Ragas version: {} is not supported)llm_factory)RagasoutputParser)context_precision)	ensembler)faithfulness)get_segmenter)answer_relevancy)embedding_factory)ContextPrecisionVerification)StatementFaithfulnessAnswers)StatementsAnswers)AnswerRelevanceClassification)ragas__version__ragas_versionr	   NotImplementedErrorformat
ragas.llmsr   ragas.llms.output_parserr   ragas.metricsr   ragas.metrics.baser   r   r   r   ragas.embeddingsr   'ddtrace.llmobs._evaluators.ragas.modelsr   r   r   r   )selfr   parsed_versionr   r   r   r   r   r   r   r   r   r   r   r    r*   Y/home/ubuntu/.local/lib/python3.10/site-packages/ddtrace/llmobs/_evaluators/ragas/base.py__init__   s>   


zRagasDependencies.__init__)r   N)__name__
__module____qualname____doc__r,   r*   r*   r*   r+   r      s    r   
span_eventr   c                 C   sN   |  dg }d}|D ]}t|tr|dr|dd } nq
|p&tjp&dS )z
    The `ml_app` spans generated from traces of ragas will be named as `dd-ragas-<ml_app>`
    or `dd-ragas` if `ml_app` is not present in the span event.
    tagsNzml_app::r   zunknown-ml-app)get
isinstancestr
startswithsplitr   _llmobs_ml_app)r1   r2   ml_apptagr*   r*   r+   _get_ml_app_for_ragas_traceT   s   r<   c                   @   sj   e Zd ZdZdZdZdd ZdefddZded	e	e
eef ee f fd
dZded	ee fddZdS )BaseRagasEvaluatora%  A class used by EvaluatorRunner to conduct ragas evaluations
    on LLM Observability span events. The job of an Evaluator is to take a span and
    submit evaluation metrics based on the span's attributes.

    Extenders of this class should only need to implement the `evaluate` method.
    r   scorec                 C   s  || _ d| _d}zzt | _| jj| _W nY ty+ } zd}td| j|d}~w tyA } zd}td| j|d}~w tyW } zd}td| j|d}~w t	ym } zd}td| j|d}~ww W t
jtjd	d
d| jfd|fd| jffd |dkrt
jtjdt d| jid dS dS t
jtjd	d
d| jfd|fd| jffd |dkrt
jtjdt d| jid w w )a  
        Initialize an evaluator that uses the ragas library to generate a score on finished LLM spans.

        :param llmobs_service: An instance of the LLM Observability service used for tracing the evaluation and
                                      submitting evaluation metrics.

        Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported.
        unknownokfail_import_errorz.Failed to load dependencies for `{}` evaluatorNfail_attribute_errorfail_not_supportedfail_unknownzevaluators.initr   evaluator_labelstateevaluator_version)	namespacenamevaluer2   z#Failed to import Ragas dependencies)levelmessagestack_tracer2   )llmobs_servicer   r   ragas_dependenciesImportErrorr    r!   LABELAttributeError	Exceptionr   add_count_metricr   MLOBSadd_logr   ERROR	traceback
format_exc)r(   rN   telemetry_stateer*   r*   r+   r,   m   st   	


zBaseRagasEvaluator.__init__r1   c              	   C   s   |sd S |  |\}}tjtjddd| jfdt|tr|ndfd| jffd t|t	rD| j
j|d|d	d
| j| j||d d S d S )Nzevaluators.runr   rE   rF   successrG   )r2   trace_idspan_id)r]   r^   )spanlabelmetric_typerJ   metadata)evaluater   rT   r   rU   rQ   r5   r6   r   floatrN   submit_evaluationr4   METRIC_TYPE)r(   r1   score_result_or_failuremetric_metadatar*   r*   r+   run_and_submit_evaluation   s*   


z,BaseRagasEvaluator.run_and_submit_evaluationr   c                 C   s   t d)Nz<evaluate method must be implemented by individual evaluators)r    )r(   r1   r*   r*   r+   rc      s   zBaseRagasEvaluator.evaluatec                    s  | j d}| j j||d d\}}}|d}|du r&	 W d   dS |d}|d}|r4|s=	 W d   dS |d}	|	du rTtd	 	 W d   dS |	d
 |d}
|d}|durtt|dkrt|d d} r|	tdg}|	tdg} fdd|D }d	 fdd|D }|s|
durt|
dkr|
d d}| j j||||dd t
dd |||fD rtd 	 W d   dS |||dW  d   S 1 sw   Y  dS )zs
        Extracts the question, answer, and context used as inputs for a ragas evaluation on a span event.
        z,dd-ragas.extract_evaluation_inputs_from_span)r_   
input_data)NNNmetaNinputoutputpromptz9Failed to extract `prompt` from span for ragas evaluation	variablesmessagesr   contentcontextquestionc                        g | ]}  |r  |qS r*   r4   .0keyprompt_variablesr*   r+   
<listcomp>        zKBaseRagasEvaluator._extract_evaluation_inputs_from_span.<locals>.<listcomp> c                    ru   r*   rv   rw   rz   r*   r+   r|      r}   )rt   contextsanswer)r_   output_datac                 s   s    | ]}|d u V  qd S )Nr*   )rx   fieldr*   r*   r+   	<genexpr>   s    zJBaseRagasEvaluator._extract_evaluation_inputs_from_span.<locals>.<genexpr>z6Failed to extract inputs required for ragas evaluation)rN   workflowannotater4   loggerdebuglenr
   r   joinany)r(   r1   extract_inputs_workflowrt   r   r   meta_io
meta_inputmeta_outputrn   input_messagesrp   context_keysquestion_keysr*   rz   r+   $_extract_evaluation_inputs_from_span   sL   










*$z7BaseRagasEvaluator._extract_evaluation_inputs_from_spanN)r-   r.   r/   r0   rQ   rf   r,   dictri   tupler   rd   r6   r   rc   r   r*   r*   r*   r+   r=   b   s    .&r=   )rX   typingr   r   ddtracer   ddtrace.internal.loggerr   ddtrace.internal.telemetryr   $ddtrace.internal.telemetry.constantsr   r   ddtrace.internal.utils.versionr	   ddtrace.llmobs._constantsr
   r   r-   r   r   r   r6   r<   r=   r*   r*   r*   r+   <module>   s    B