o پi3 @s6ddlZddlZddlZddlmZddlmZmZddlm Z ddl m Z m Z m Z mZmZddlZddlZddlZddlZddlZddlmZddlmZdZd Ze ee fZe eZGd d d ZeGd d d ZeGdddZGdddZ Gdddej!Z"GdddeZ#d$Z%dZ&dZ'd$Z(dZ)ddZ*dededefd d!Z+d"e,d#efd$d%Z- & dEd'e ed(eed)ee eeefd*efd+d,Z.d-e/d.e e d/e0fd0d1Z1ej2e3ej4e5d2d3gd4Z6d5Z7d6ed*efd7d8Z8e8e6j9d8<d9Z:d:ed*efd;d<Z;d=e efd>d?ZdS)GN) defaultdict) dataclassfield) ThreadPool)AnyDictListOptionalTuple)OpenAI)tqdmzYou are a helpful assistant.zYou are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture. Knowledge cutoff: 2023-12 Current date: 2024-04-01c@"eZdZdZdedefddZdS) SamplerBasezw Base class for defining a sampling model, which can be evaluated, or used as part of the grading process. message_listreturncCtNNotImplementedError)selfrrR/home/ubuntu/.local/lib/python3.10/site-packages/sglang/test/simple_eval_common.py__call__$zSamplerBase.__call__N)__name__ __module__ __qualname____doc__ MessageListstrrrrrrrsrc@sJeZdZUdZeeed<eeeefed<e eed<e e ed<dS) EvalResultzN Result of running an evaluation (usually consisting of many samples) scoremetricshtmlsconvosN) rrrrr float__annotations__rrrrrrrrr (s   r c@sXeZdZUdZeeed<eedZ e e efed<dZ ee ed<dZ eeed<dS)SingleEvalResultz. Result of evaluating a single sample r!)default_factoryr"Nhtmlconvo)rrrrr r%r&rdictr"rrr)r*rrrrrr'4s  r'c@r )Evalz0 Base class for defining an evaluation. samplerrcCrrr)rr-rrrrErz Eval.__call__N)rrrrrr rrrrrr,@sr,cseZdZfddZZS)LargerHttpxClientcs,td}tjddd}tj||ddS)Ni)max_keepalive_connectionsmax_connections)timeoutlimits)httpxTimeoutLimitssuper__init__)rtimeout_configr2 __class__rrr7Js zLargerHttpxClient.__init__)rrrr7 __classcell__rrr9rr.Isr.c@seZdZdZ        d$dedeedeed ed ed eed ed eeee ffddZ   d%dedededefddZ defddZ dede fddZ d ed!efd"d#ZdS)&ChatCompletionSamplerz2 Sample from OpenAI's chat completion API N?base_urlmodelsystem_message temperaturetop_preasoning_effort max_tokens extra_bodyc Cst|td|_|dur|jjjdj}||_||_||_ ||_ ||_ ||_ ||_ d|_td|jd|j d|j d|j d|j  dS) N)r@ http_clientrurlz;ChatCompletionSampler initialized with self.system_message=z self.temperature=z self.max_tokens=z self.reasoning_effort=z self.extra_body=)r r.clientmodelslistdataidrArBrCrDrFrErG image_formatprint) rr@rArBrCrDrErFrGrrrr7Xs *zChatCompletionSampler.__init__base64pngimageencodingformatfoveacCs$ddd|d|d|id}|S)N image_urlrIz data:image/;,)typerXr)rrTrUrVrW new_imagerrr _handle_imagets z#ChatCompletionSampler._handle_imagetextcCs d|dS)Nr^)r[r^r)rr^rrr _handle_texts z"ChatCompletionSampler._handle_textrolecontentcCst||dS)N)r`ra)r)rr`rarrr _pack_messagesz#ChatCompletionSampler._pack_messagerrc Cs|jr |d|jg|}d}|dkrzz|jjjj|j||j|j|j |j |j d}|j dj jp1dWStjyK}z td|WYd}~dSd}~wtyu}zd|}td|d |d |t||d 7}WYd}~nd}~ww|dkstd dS) Nsystemr)rAmessagesrCrDrFrErGzBad Request Errorz'Rate limit exception so wait and retry z after z seczCAll retry attempts exhausted for request. Returning empty response.)rBrbrJchat completionscreaterArCrDrFrErGchoicesmessageraopenaiBadRequestErrorrP Exceptiontimesleep)rrtrialresponseeexception_backoffrrrrsF     zChatCompletionSampler.__call__)NNNr=r>Nr?N)rQrRrS)rrrrrr r%intrrr7r]r_rbrrrrrrr<SsT  r<a Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. {Question} A) {A} B) {B} C) {C} D) {D} z(?i)Answer\s*:\s*([A-D])z(?i)Answer\s*:\s*([^\n]+)a Look at the following two expressions (answers to a math problem) and judge whether they are equivalent. Only perform trivial simplifications Examples: Expression 1: $2x+3$ Expression 2: $3+2x$ Yes Expression 1: 3/2 Expression 2: 1.5 Yes Expression 1: $x^2+2x+1$ Expression 2: $y^2+2y+1$ No Expression 1: $x^2+2x+1$ Expression 2: $(x+1)^2$ Yes Expression 1: 3245/5 Expression 2: 649 No (these are actually equal, don't mark them equivalent if you need to do nontrivial simplifications) Expression 1: 2/(-3) Expression 2: -2/3 Yes (trivial simplifications are allowed) Expression 1: 72 degrees Expression 2: 72 Yes (give benefit of the doubt to units) Expression 1: 64 Expression 2: 64 square feet Yes (give benefit of the doubt to units) --- YOUR TASK Respond with only "Yes" or "No" (without quotes). Do not include a rationale. Expression 1: %(expression1)s Expression 2: %(expression2)s aA

Prompt conversation

{% for message in prompt_messages %} {{ message_to_html(message) | safe }} {% endfor %}

Sampled message

{{ message_to_html(next_message) | safe }}

Results

Correct Answer: {{ correct_answer }}

Extracted Answer: {{ extracted_answer }}

Score: {{ score }}

cCstjdi|S)Nr)QUERY_TEMPLATE_MULTICHOICErV)rowrrrformat_multichoice_questionsrzr-expr1expr2cCs4t||d}|t|ddg}|pddkS)N) expression1 expression2user)rar`rfyes)EQUALITY_TEMPLATEr+lowerstrip)r-r{r|promptrtrrrcheck_equality srvaluesstatcCsV|dkr t|S|dkrt|S|dkrt|S|dkr$t|Std|)NmeanstdminmaxzUnknown stat =)nprrrr ValueError)rrrrr _compute_stats    rrrsingle_eval_results default_stats name2statsrcCs|pi}tt}g}g}|D]1}|durq|jD] \}}|||q|jdur3|d|j||j||jqi} |D]#\}} |||} | D]} | dkrZ|n|d| } t | | | | <qRqFt | dd| ||dS)zO Aggregate results from multiple evaluations into a single EvalResult. Nr!r:)r!r"r#r$) rrLr"itemsappendr!r)r*getrr pop)rrr name2valuesr#r$single_eval_resultnamevalue final_metricsrstatsrkeyrrraggregate_resultss4    rfxs num_threadscCsvtdrtt|t|t|dStt|t|}tt|||t|dWdS1s4wYdS)zO Apply f to each element of xs, using a ThreadPool, and show progress. debug)totalN) osgetenvrLmapr lenrrimap)rrrpoolrrrmap_with_progressAs $rr)xml)loader undefined autoescapez
{{ role }} {% if variant %}({{ variant }}){% endif %} 
{{ content }}
rmcCs&ttj|d|d|dddS)z? Generate HTML snippet (inside a
) for a message. r`ravariantN)r`rar) jinja_env from_string_message_templaterenderr)rmrrrmessage_to_html^s  ra {% if metrics %}

Metrics

{% for name, value in metrics.items() %} {% endfor %}
Metric Value
Score {{ score | float | round(3) }}
{{ name }} {{ value }}
{% endif %}

Examples

{% for html in htmls %} {{ html | safe }}
{% endfor %} eval_resultcCsttj|j|j|jdS)z= Create a standalone HTML report from an EvalResult. r!r"r#)rr_report_templaterr!r"r#)rrrr make_reports rr#cCsttjdi|dS)zF Create a standalone HTML report from a list of example htmls Nr)rrrr)r#rrrmake_report_from_example_htmlss rc Cstd|d|z`tj|dd}|t|jdd}d}t|d3}td |d dd d }||D] }| |}| |q6Wdn1sMwYWdn1s\wYtd |WdStj y~} zt d| d} ~ ww)NzDownloading dataset z from T)streamzcontent-lengthri wb DownloadingiBi)descrunit unit_scale unit_divisorz Dataset downloaded and saved to zFailed to download dataset: ) rPrequestsrraise_for_statusrwheadersopenr iter_contentwriteupdateRequestExceptionrp) pathrIrt total_size block_sizer progress_barrMsizerurrrdownload_datasets4   rc Csltj}t|\}}||kr4z t|||fWdSty3}ztd|WYd}~dSd}~wwdS)NzFail to set RLIMIT_NOFILE: )resource RLIMIT_NOFILE getrlimit setrlimitrrP)target_soft_limit resource_type current_soft current_hardrurrr set_ulimitsr)rN)r)?rrrq collectionsr dataclassesrrmultiprocessing.poolrtypingrrrr r r3jinja2numpyrrnrr r OPENAI_SYSTEM_MESSAGE_APIOPENAI_SYSTEM_MESSAGE_CHATGPTrMessagerrr r'r,Clientr.r<rrxANSWER_PATTERN_MULTICHOICEANSWER_PATTERNr HTML_JINJArzrrLrrcallablerwr Environment BaseLoaderStrictUndefinedselect_autoescaperrrglobalsrrrrrrrrrs          Y  := $   B