o
    wi                     @   sN   d dl Z d dlmZmZ d dlmZ G dd de jZedkr%e 	  dS dS )    N)	MagicMockpatch)evaluatec                   @   s   e Zd Zdd Zededededdd Zededededd	d
 Zededededdd ZdS )TestEvaluateFunctionc                 C   s   t  | _t  | j_d| jj_d| jj_d| jj_d| jj_t  | _d| j_t  | j_	d| jj	_
d| jj	_d| jj	_d	| jj	_d
| jj	_d| jj	_d| jj	_d| jj	_d| jj	_d S )Nzpath/to/checkpointhttp://example.com@  model_idgsm8k            ??2   Td        )r   
target_cfgapi_endpointnemo_checkpoint_pathurlnemo_triton_http_portr   eval_cfgtypeparams
batch_sizemax_new_tokenstemperaturetop_ptop_kadd_boslimit_samplesnum_fewshotbootstrap_iters)self r$   b/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/tests/evaluation/eval_unittest_legacy.pysetUp   s$   













zTestEvaluateFunction.setUpznemo.lightning.io.load_contextz:nemo.collections.llm.evaluation.base.wait_for_server_readyz!lm_eval.evaluator.simple_evaluatez1nemo.collections.llm.evaluation.base.NeMoFWLMEvalc                 C   sz   d|_ d|_ dddii|_ t| j| j |jddd |jd	d
dd |jdd	dddddddd	 |jdddddd d S )N	tokenizermodelresultsr	   scorezpath/to/checkpoint/contextzmodel.tokenizer)subpathr   r   r   )r   triton_http_port
model_namer
   r   r   r   r   T)	r-   api_urlr'   r   max_tokens_to_generater   r   r   r   r   r   r   )r(   taskslimitr!   r"   )return_valuer   r   r   assert_called_once_withr#   mock_NeMoFWLMEvalmock_simple_evaluatemock_wait_for_server_readymock_load_contextr$   r$   r%   test_evaluate_success-   s4   
z*TestEvaluateFunction.test_evaluate_successc                 C   sf   d | j j_| t t| j | j W d    n1 sw   Y  |  |  |  |  d S )N)r   r   r   assertRaises
ValueErrorr   r   assert_not_calledr4   r$   r$   r%   'test_evaluate_nemo_checkpoint_path_noneT   s   
z<TestEvaluateFunction.test_evaluate_nemo_checkpoint_path_nonec              	   C   s   t dtdd5 | t t| j| j W d    n1 s w   Y  |  |  |  |  W d    d S 1 s@w   Y  d S )Nzbuiltins.__import__zMocked ImportError)side_effect)r   ImportErrorr:   r   r   r   r<   r4   r$   r$   r%   test_evaluate_import_errorh   s   
"z/TestEvaluateFunction.test_evaluate_import_errorN)__name__
__module____qualname__r&   r   r9   r=   r@   r$   r$   r$   r%   r      s"    #r   __main__)
unittestunittest.mockr   r   nemo.collections.llm.apir   TestCaser   rA   mainr$   r$   r$   r%   <module>   s   g