o
    wi                     @   s,  d dl Z d dlZd dlZd dlmZ d dlmZmZmZm	Z	 d dl
mZ d dlmZ dd Zdd	 Zed
kre ZeeZed edddZered eddZe	edZdejiZejdurkdejded< eejedi edZeeed ed eej dS eej eddS )    N)evaluate)ApiEndpointConfigParamsEvaluationConfigEvaluationTarget)wait_for_fastapi_server)loggingc                  C   sv   t jdd} | jdtdd | jdtd dd | jd	td
d | jdtdd | jdtdd | jdddd |  S )NzMTest evaluation with NVIDIA Evals Factory on nemo2 model deployed on PyTriton)description--nemo2_ckpt_pathzNeMo 2.0 ckpt path)typehelpz--tokenizer_pathzPath to the tokenizer)r   defaultr   --max_batch_sizez#Max BS for the model for deploymentz--eval_typez5Evaluation benchmark to run from NVIDIA Evals Factoryz--limitz*Limit evaluation to `limit` num of samples--legacy_ckpt
store_truez/Whether the nemo checkpoint is in legacy format)actionr   )argparseArgumentParseradd_argumentstrint
parse_args)parser r   ]/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/tests/evaluation/test_evaluation.pyget_args   s   r   c              	   C   s2   t ddd| jdt| jg| jrdg S g  S )Npythonz'tests/evaluation/deploy_in_fw_script.pyr
   r   r   )
subprocessPopennemo2_ckpt_pathr   max_batch_sizelegacy_ckpt)argsr   r   r   
run_deploy'   s   
	r#   __main__zWaiting for server readiness...zhttp://0.0.0.0:8886x   )base_urlmax_retrieszStarting evaluation...z#http://0.0.0.0:8886/v1/completions/)url)api_endpointlimit_sampleshuggingface)tokenizer_backend	tokenizerextra)r   params)
target_cfgeval_cfgzEvaluation completed.zEServer is not ready. Please look the deploy process log for the errorr   )r   signalr   nemo.collections.llmr   #nemo.collections.llm.evaluation.apir   r   r   r   $nemo.collections.llm.evaluation.baser   
nemo.utilsr   r   r#   __name__r"   deploy_procinfoserver_readyr)   eval_targetlimiteval_paramstokenizer_path	eval_typeeval_configsend_signalSIGINTRuntimeErrorr   r   r   r   <module>   s>   






