o
    پi                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
mZmZ d dlmZmZ dZdZdd	 Zd
d Zdd ZG dd de	ZdS )    N)Optional)simple_eval_common)
HTML_JINJAEval
EvalResultSamplerBaseSingleEvalResult)download_and_cache_file
read_jsonlzchttps://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonliigc                 C   s2   d| | d  d}|r|d| | d  7 }|S )Nz
Question: questionz
Answer: answer )linesiinclude_answerretr   r   Q/home/ubuntu/.local/lib/python3.10/site-packages/sglang/test/simple_eval_gsm8k.pyget_one_example   s   r   c                    s   d  fddt|D S )N c                 3   s     | ]}t  |d d V  qdS )Tz

N)r   ).0r   r   r   r   	<genexpr>   s    z(get_few_shot_examples.<locals>.<genexpr>)joinrange)r   kr   r   r   get_few_shot_examples   s   r   c              	   C   sT   |  dd} td| }t|dk rtS zt|d W S  ttfy)   t Y S w )N,r   z-?\d+\.?\d*   )	replacerefindalllenINVALIDastliteral_evalSyntaxError
ValueError)
answer_strnumbersr   r   r   get_answer_value    s   r+   c                
   @   sJ   e Zd Z				ddee dededee fdd	Zd
edefddZ	dS )	GSM8KEvalN@      num_examplesnum_threads	num_shots	data_pathc                 C   sj   || _ || _|r|}ntt}tt|| _t| j|| _| j|d  | _|d ur3| jd | | _d S d S )N)	_num_threads
_num_shotsr	   	GSM8K_URLlistr
   _linesr   _few_shot_prompt)selfr/   r0   r1   r2   filenamer   r   r   __init__,   s   zGSM8KEval.__init__samplerreturnc                    sF   dt dtf fdd}tj|tttjjd}tj	|ddS )Nidxr=   c           
         s   t j| dd}tj|  d }j| } j|ddg}z |}W n ty/   d}Y nw t|}t||k}tj	t
j|t|dd|||d}|t|ddg }	t|||	d	S )
NF)r   r   user)contentroler   	assistant)prompt_messagesnext_messagescorecorrect_answerextracted_answer)htmlrE   convo)r   r7   r+   r8   _pack_message	Exceptionfloatcommon	jinja_envfrom_stringr   renderdictr   )
r>   r   rF   prompt_contentrC   response_textrG   rE   rH   rI   r<   r9   r   r   fnD   s*   

zGSM8KEval.__call__.<locals>.fn)r0   )meanstd)default_stats)
intr   rM   map_with_progressr6   r   r#   r7   r3   aggregate_results)r9   r<   rU   resultsr   rT   r   __call__C   s
   zGSM8KEval.__call__)Nr-   r.   N)
__name__
__module____qualname__r   rY   strr;   r   r   r]   r   r   r   r   r,   +   s    
r,   )r%   r!   typingr   sglang.testr   rM   sglang.test.simple_eval_commonr   r   r   r   r   sglang.utilsr	   r
   r5   r$   r   r   r+   r,   r   r   r   r   <module>   s   