o
    }oi                     @   s   d dl Z d dlmZ d dlmZ d dlmZmZmZ e j	dddd Z
e jd	d
ddddidd
iddidddddddddgd	efddZdd Ze jdg ddedefd d!ZdS )"    N)
HTTPServer)evaluate)ConfigParamsEvaluationConfigEvaluationTargetsession)scopec                   C   s   dS )N)z	127.0.0.1i@   r	   r	   r	   R/home/ubuntu/.local/lib/python3.10/site-packages/tests/evaluation/eval_unittest.pyhttpserver_listen_address   s   r   paramsg?gMbP?)top_ptemperaturelimit_samples
   max_new_tokens@      d   )max_retriesparallelismrequest_timeoutmy_task   my_tokenizer)num_fewshot	tokenizer)taskextrac                 C   sT   t d| d}t|jtsJ |jdksJ |  D ]\}}t|j||ks'J qd S )Ncustomtyper   )r   
isinstancer   r   r!   itemsgetattr)r   eval_config
param_nameparam_valuer	   r	   r
   test_configuration   s   r(   c                  C   sN   t ddddiid} | jdksJ | jjd d u sJ | jjd dks%J d S )Nr   r   r   r   r    r   )r   r!   r   r   )r%   r	   r	   r
   test_default_none_tokenizer1   s   r)   r   )gsm8kzlm-evaluation-harness.gsm8kzlm_evaluation_harness.gsm8k
httpserverc                 C   s   |  dddi | j dddddd	d
ddigd tdddd}t|tdddd}t||d}|d d d d d d d dksIJ d S )Nz/v1/triton_healthstatusz$Triton server is reachable and readyz/v1/completions/POST)methodzcmpl-123456text_completioni triton_modeltextz Janet eats 3 eggs and bakes 4 eggs, so she has 16 - 3 - 4 = <<16-3-4=9>>9 eggs left.
She sells 9 eggs for $2 each, so she makes 9 x 2 = <<9*2=18>>18 dollars.
#### 18)idobjectcreatedmodelchoicesz%http://localhost:8000/v1/completions/completions)urlr!   )api_endpoint   )r   r   r    )
target_cfgeval_cfgtasksr*   metricszexact_match__strict-matchscoresvalueg      ?)expect_requestrespond_with_jsonr   r   r   r   )r+   r   target_configr%   resultsr	   r	   r
   test_evaluation8   s4   

rE   )pytestpytest_httpserverr   nemo.collections.llm.apir   #nemo.collections.llm.evaluation.apir   r   r   fixturer   markparametrizedictr(   r)   strrE   r	   r	   r	   r
   <module>   s,   


