o 5ti4@sddlZddlZddlmZddlmZddlmZmZm Z m Z m Z m Z ddl mZddlmZddlmZeeZedGd d d eZed Gd d d eZedGdddeZedGdddeZedGdddeZdS)N)cached_property) itemgetter)AnyDictListOptionalTupleUnion)register_model) TemplateAPI)handle_stop_sequenceszlocal-completionsc seZdZ     dfdd Z    ddeeeeeeeeefd e ed ed efd d Z e  ddee ee fdeeedeed ee eeffddZe dee ee fd eefddZeddZZS)LocalCompletionsAPINautoTc sx|dkr+|r$ddlm}|||||drtdd}ntdd}ntd d}tjd |||||d |dS) Nrr)check_remote_tokenizer_support)verify_certificate ca_cert_path auth_tokenzGAuto-detected remote tokenizer support. Using remote tokenizer backend.remotezDRemote tokenizer not supported. Using huggingface tokenizer backend. huggingfacez:No base_url provided. Using huggingface tokenizer backend.)base_urltokenizer_backendrrr) lm_eval.utilsr eval_loggerinfowarningsuper__init__)selfrrrrrkwargsr __class__rU/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/models/openai_completions.pyrs<   zLocalCompletionsAPI.__init__Fmessages gen_kwargsseedreturnc Ks~|r4|ddd|vr|d}n|d|j}|dd}t|dd|} ||j||| |d|S|j|dd d |d d S) N do_sampleF max_tokens max_gen_toks temperatureruntil)promptmodelr)r+stopr&T)r.r-r+r)logprobsr&echo)pop _max_gen_toksr r. rr$generater%r&eosrr)r+r/rrr"_create_payload=s0    z#LocalCompletionsAPI._create_payloadoutputstokensctxlenscKsg}t|ts |g}|D][}tt|dtdd|D]K\}}|dks'Jdt|dd|d}|dd|d} |dd |d} d } t| | D]\} } | t| kr^d } nqN||| fqq |S) Nchoicesindex)keyrz%Context length must be greater than 0r1token_logprobs top_logprobsTF) isinstancelistzipsortedrsummaxvaluesappend)r9r:r;rresoutchoicectxlenr1tokens_logprobsrA is_greedytoktoprrr"parse_logprobsbs(   z"LocalCompletionsAPI.parse_logprobscKsZg}t|ts |g}|D]}dgt|d}|dD] }|d||d<q||}q |S)Nr<textr=)rBrClen)r9rrJrKtmpr<rrr"parse_generations|s   z%LocalCompletionsAPI.parse_generationscCstjddS)NOPENAI_API_KEY)osenvironget)rrrr"api_keyszLocalCompletionsAPI.api_key)NrTNNFNr#NNN)__name__ __module__ __qualname__rr rintdictstrrr8 staticmethodrrfloatboolrRrVpropertyr\ __classcell__rrr r"r sH/ %  $ r zlocal-chat-completionscseZdZdZ      dfdd Z    ddeed ed efd d Ze d e eeefd ee fddZ  dde e e fd e ee eee ffddZddZZS)LocalChatCompletionz Minimal chat-completions wrapper. - Only accepts messages as list[dict]. - No tokenization or template logic. - Use with --apply_chat_template or ensure upstream formats messages correctly. NTc sBtjd||||||d||jdkrtdd|_dSdS)N)rrtokenized_requestsrrrr0zGChat completions does not support batching. Defaulting to batch size 1.r)rr _batch_sizerr)rrrrkrrrrr rr"rs    zLocalChatCompletion.__init__Fr#r$r%r'c Kst|trtdd|DsJd|pi}|ddd|vr&|d}n|d|j}|dd }t|d d|} t| ttfsF| g} ||j||| dd |d |S) Ncss|]}t|tVqdSN)rBrc).0mrrr" s  z6LocalChatCompletion._create_payload..zLocalChatCompletion expects messages as list[dict]. If you see this error, ensure --apply_chat_template is set or upstream code formats messages correctly.r(Fr)r*r+rr,)r$r.r)r+r/r&)rBrCallr3r4r tupler.r5rrr"r8s.      z#LocalChatCompletion._create_payloadr9c Ksg}t|ts |g}|D]@}zdgt|d}|dD] }|dd||d<qWntyG}ztd|dg}WYd}~nd}~ww||}q |S)Nr<messagecontentr=zCould not parse generations: rX)rBrCrT Exceptionrr)r9rrJrKrUr<errr"rVs    z%LocalChatCompletion.parse_generationsstringcKs|Srmr)rrxleft_truncate_lenadd_special_tokensrrrr" tok_encodeszLocalChatCompletion.tok_encodecKtd)Nz`Loglikelihood is not supported for chat completions. Consider using the completions API instead.NotImplementedErrorrrequestsrrrr" loglikelihoodz!LocalChatCompletion.loglikelihood)NNNTNNr]r^)r_r`ra__doc__rrrrcr8rer rdrVrrbr{rrirrr r"rjs<  #$   rjzopenai-completionscsZeZdZ  dfdd ZeddZfddZdd eee fd e e fd d Z Z S)OpenAICompletionsAPI%https://api.openai.com/v1/completionstiktokenc stjd||d|dS)N)rrr)rr)rrrrr rr"rs  zOpenAICompletionsAPI.__init__cC"tjdd}|durtd|SzAOverride this property to return the API key for the API request.rWNzHAPI key not found. Please set the `OPENAI_API_KEY` environment variable.rYrZr[ ValueErrorrr>rrr"r\ zOpenAICompletionsAPI.api_keyc s2|jdvsJdddgdtj|fi|S)N) babbage-002 davinci-002z=Prompt loglikelihoods are only supported by OpenAI's API for rr.)r.rrrr rr"r s z"OpenAICompletionsAPI.loglikelihoodF chat_templater'cCsdS)NrXr)rrrrr"rsz"OpenAICompletionsAPI.chat_template)rr)F) r_r`rarrr\rr rgrdrrrirrr r"rs  ( rzopenai-chat-completionscs\eZdZ   dfdd ZeddZdd Z   dd eed e de fddZ Z S)OpenAIChatCompletion*https://api.openai.com/v1/chat/completionsNFc s8d|ddvr tdtjd|||d|dS)No1r.rXz>o1 models do not support `stop` and only support temperature=1)rrrkr)r[rrrr)rrrrkrr rr"rs zOpenAIChatCompletion.__init__cCrrrrrrr"r\.rzOpenAIChatCompletion.api_keycKr|)NaRLoglikelihood (and therefore `multiple_choice`-type tasks) is not supported for chat completions as OpenAI does not provide prompt logprobs. See https://github.com/EleutherAI/lm-evaluation-harness/issues/942#issuecomment-1777836312 or https://github.com/EleutherAI/lm-evaluation-harness/issues/1196 for more background on this limitation.r}rrrr"r8rz"OpenAIChatCompletion.loglikelihoodr# <|endoftext|>r$r%r'c Kst|tus Jd|ddd|vr|d}n|d|j}|dd}t|dd g|} t| ttfs;| g} ||j||| dd |d |} d |jvs_d |jvs_d|jvs_d|jvrh| dd| d<| S)Nz8chat-completions require the --apply_chat_template flag.r(Fr)r*r+rr,rrq)r$r.max_completion_tokensr+r/r&r5o3o4r/r0) typerdr3r4r rBrCrsr.) rr$r6r%r&r7rr)r+r/outputrrr"r8=s6         z$OpenAIChatCompletion._create_payload)rNF)FNr#r) r_r`rarrr\rrrrcr8rirrr r"rs&  rzazure-openai-chat-completionsc s\eZdZedededddfdededed ed d f fd d ZeddZ Z S)AzureOpenaiChatCompletionsLMAZURE_OPENAI_DEPLOYMENT_NAMEAZURE_OPENAI_ENDPOINTAZURE_OPENAI_API_VERSIONz2025-03-01-previewFr.r api_versiontruncater'Nc shtzddl}Wn tytdw||_|d|d||_||_|j|||j d|_ dS)Nrzattempted to use 'openai' LM type, but package `openai` or `tiktoken` are not installed. please install these via `pip install lm-eval[openai]` or `pip install -e .[openai]`z/openai/deployments/z/chat/completions?api-version=)azure_endpointrr\) rropenaiModuleNotFoundErrorrvr.rr AzureOpenAIr\client)rr.rrrrrr rr"rhs    z%AzureOpenaiChatCompletionsLM.__init__cCs"tjdd}|durtd|S)NAZURE_OPENAI_API_KEYzNAPI key not found. Please set the `AZURE_OPENAI_API_KEY` environment variable.rrrrr"r\s z$AzureOpenaiChatCompletionsLM.api_key) r_r`rarYgetenvrdrgrrr\rirrr r"rfs$ r)loggingrY functoolsroperatorrtypingrrrrrr lm_eval.api.registryr lm_eval.models.api_modelsr lm_eval.models.utilsr getLoggerr_rr rjrrrrrrr"s*       }f"L