o 5tiS@sddlZddlZddlmZddlmZmZmZmZm Z m Z ddl m Z ddl m Z ddlmZddlmZddlmZmZmZdd lmZmZeeZzddlZWn ey\Ynwer` ed Gd d d eZdS) N) find_spec) TYPE_CHECKINGDictListOptionalTupleUnion)tqdm)Instance) TemplateLM)register_model)Collatorhandle_stop_sequencespostprocess_generated_text)get_rolling_token_windowsmake_disjoint_windowsglangc(seZdZdZ                  dOd ed eeefd ed edeedeededededededeedededee dededeedeef&fdd Z dPde e d ed!e e fd"d#Z dPde e d ed!e efd$d%Z      dQde e ed&ed'ee eedfd(ed)ed*ef d+d,Zed-d.Zed/d0Zed1d2Zed3d4Z   dRd5eee efd6ed7ed8ed!ee ee e eff d9d:Zd;e ed!efdd?ZdPd@eeefd!efdAdBZ dSdCe eeefdDed!efdEdFZ dPde eeeefe ee efd ed!e ee effdGdHZed;e dIed!ee effdJdKZedLed!efdMdNZ Z!S)TSGLangLMiNFautoTcuda pretrained batch_size max_model_len max_gen_toks add_bos_tokentokenizer_pathtokenizer_mode load_formattrust_remote_codedtypekv_cache_dtypecontext_lengthdevicechunked_prefill_sizemem_fraction_staticdp_sizetp_sizeprefix_token_idthink_end_tokenc s,ttds tdd|vs|dusJd| dus%|dus%Jd||_|dur.|n| |_t||_t||_|||| | | | |||j|j|d |_ |j |t |t r^d|vr^dnt||_ |jdkrmtd tjd i|j |_|jjj|_||_||_d |vrd |_td ||_dS)Nrzattempted to use 'sglang' LM type, but package `sglang` is not installed. Please install sglang via official document here:https://docs.sglang.ai/start/install.html#install-sglangrzSGLang only supports CUDAzDEither context_length or max_model_len may be provided, but not both) model_pathrrr r!r"r#r%r'r)r(r&rrzData parallelism will be deprecated in the future version of SGLang. See here: https://docs.sglang.ai/backend/server_arguments.html#data-parallelism .gemmaTzeFound 'gemma' in model name, a BOS token will be used as Gemma series models underperform without it.)super__init__rModuleNotFoundErrorr+ _max_lengthinttensor_parallel_sizedata_parallel_size model_argsupdate isinstancestrr eval_loggerwarningsglEnginemodeltokenizer_manager tokenizer _max_gen_toksrlowerinfocustom_prefix_token_id)selfrrmax_batch_sizerrrrrr r!r"r#r$r%r&r'r(r)r*r+kwargs __class__r.S/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/models/sglang_causallms.pyr0%s`        zSGLangLM.__init__requests disable_tqdmreturnc s~d}|jdkr t|}g}g}ttdd|D|p|jdkdD]3\\}tttt| ||j |j ddd}dd|D}| fd d |D| t|q!g} |p]t|j} tdt|| D]} || | | } t| \} }|j|d d }| t| |qfg}d}|D]0}| |||}td d |D}| |||7}|t|djd}|jd|f|q|S)NrcSsg|]}|jqSr.args.0reqr.r.rJ sz2SGLangLM.loglikelihood_rolling..r)disabler) token_list prefix_token max_seq_len context_lencSsg|]}d|qS)Nr.)rQxr.r.rJrSsc3s|]}|fVqdSrYr.)rQwindowreq_idxr.rJ sz1SGLangLM.loglikelihood_rolling..F)rKrLcss|] \}}|dVqdS)rNr.)rQ_nllr.r.rJr^sloglikelihood_rolling)rlen enumerater ranklistmaprr tok_encoder* max_lengthextendappendr3rangezip_loglikelihood_tokenssumrO cache_hook add_partial)rErKrLadaptive_batch_size all_windowsrequest_window_countsstringrolling_token_windowswindowsall_nllsribatch batch_indices batch_windows batch_nllsloglikelihoods current_idx window_count request_nlls request_totalr.r\rJra|s^     zSGLangLM.loglikelihood_rollingcCsg}tdd|D\}}|j||jd}ddt|||D}dd}t||dd}|j|jd kr7t|jnd dd } tt||pF|j d kd d } |j |j } | D]} t| \} }t| \}}g}g}t||D][\}}t |trt|}t|dd| d}n tdt|d|vr|d}n|j}|j|}t||kr||| dn||||}||||dBqj|j|d|d}t||D]%\}}|dd}t|||j}|||jd||f|| dqqS| !|"|S)Ncss|]}|jVqdSrYrNrPr.r.rJr^sz*SGLangLM.generate_until..)add_special_tokenscSsg|] \}}}||f|fqSr.r.)rQabcr.r.rJrSsz+SGLangLM.generate_until..cSst|dd |ddfS)Nrr)rb) _requestsr.r.rJ _collate_gensz-SGLangLM.generate_until.._collate_gen)group_byrrnbatch_fnzRunning generate_until requeststotalrTdescuntil)eosz/Expected `kwargs` to be of type `dict` but got r) max_tokensstopT)rKgeneratesampling_paramstextgenerate_untilr)#rlrgrr get_batchedrr3r rbrdr@decode eot_token_idr8dictcopydeepcopyrpop ValueErrortypekeysrrhrjmodify_gen_kwargs_model_generategetrr+rorpr7close get_original)rErKrLrescontextall_gen_kwargscontext_encodingrre_ordschunkspbarrchunkcontext_and_encodingcontext_encoding_truncatedrrZ gen_kwargsrGrr max_ctx_lencontoutputgenerated_textr.r.rJrsx                    zSGLangLM.generate_untilrrreturn_logprobtop_logprobs_numlogprob_start_lencCsR|s|r|ni}|dddt|ts|gt|}|jj|||||d}|S)Nrr) temperaturemax_new_tokens) input_idsrrrr)r7r8rrbr>r)rErKrrrrroutputsr.r.rJr s"  zSGLangLM._model_generatecCs|jjSrY)r@ eos_token_idrEr.r.rJr>szSGLangLM.eot_token_idcCs,|jdur|jS|jjdur|jjS|jjSrY)rDr@ bos_token_idrrr.r.rJr*Cs  zSGLangLM.prefix_token_idcCs6|jr|jSt|jdrt|jjdr|jjjS|jS)Nr?rX)r2hasattrr>r?rX_DEFAULT_MAX_LENGTHrr.r.rJrhLs zSGLangLM.max_lengthcCs|jSrY)rArr.r.rJrVszSGLangLM.max_gen_toksrtleft_truncate_lenr truncationcsT|s|j}|j|||ddj}r(t|ts!fdd|D}|S| d}|S)NF)rrreturn_attention_maskcsg|] }| dqSrYr.)rQencrr.rJrSnsz'SGLangLM.tok_encode..)rr@rr8r9)rErtrrrencodingr.rrJrg[s  zSGLangLM.tok_encodetokenscCsdSrYr.)rErr.r.rJ tok_decodetszSGLangLM.tok_decodecCdS)z Return the name of the model's tokenizer and/or the accompanying chat template. The returned string is used to cache requests. Returns: str: The name of the model's tokenizer and/or chat template. Nr.rr.r.rJtokenizer_namexs zSGLangLM.tokenizer_name chat_templatecCr)a Get the appropriate chat template for the model based on the `chat_template` argument. This method returns the chat template string to build the prompt from a chat history. The chat template is saved in the evaluation results for reproducibility. Boolean arguments should be used with models that have only one chat template, while string arguments are used with models that have multiple chat templates. For the reference implementation, see HFLM class in `lm_eval.models.huggingface`. Args: chat_template (Union[bool, str]): Specifies whether to apply a chat template: - If False: Do not apply any chat template. - If True: Apply the default chat template. - If str: Apply the specified chat template by name. Returns: str: The selected chat template in Jinja format. Nr.)rErr.r.rJrszSGLangLM.chat_template chat_historyadd_generation_promptcCs|jj|d|| d}|S)zc Method to apply a chat template to a list of chat history between user and model. F)tokenizercontinue_final_message)r@apply_chat_template)rErrchat_templatedr.r.rJrszSGLangLM.apply_chat_templatecCs>g}dd}t||d}|j|jdkrt|jnddd}tt||dd}|D]m}g} g} |D]+\} } } | | |j d}t| tdt| t| |j}| || |q0|j | d d d dd }t || || D]'\}}\} }}}|j |||d }||| dur|j d| ||dqmq(|||S)NcSs"|d|d}t| t|fS)Nr)rbtuple)rZtoksr.r.rJ_collatesz0SGLangLM._loglikelihood_tokens.._collate)sort_fnrrrzRunning loglikelihood requestsrFTr)rKrrrr)rrctxlen loglikelihoodr)r rrr3r rbrhmaxrjrrl_parse_logprobsrorpr7rr)rErKrLrrre_ordrrrinputsctxlens cache_key context_enccontinuation_encinprrrr_answerr.r.rJrmsV      zSGLangLM._loglikelihood_tokensrc Cs|dd}tdd||dD}|dd}d}t||d||dD]\}}|rCt|dd d d } | |krCd }||fSq(||fS) aProcess logprobs and tokens. :param tokens: list Input tokens (potentially left-truncated) :param outputs: Contains input_token_logprobs and input_top_logprobs :param ctxlen: int Length of context (so we can slice them away and only keep the predictions) :return: continuation_logprobs: float Log probabilities of continuation tokens is_greedy: bool Whether argmax matches given continuation exactly meta_infoinput_token_logprobscss|]\}}}|VqdSrYr.)rQlogprobr_r.r.rJr^s  z+SGLangLM._parse_logprobs..Ninput_top_logprobsTcSs|dS)Nrr.)rZr.r.rJsz*SGLangLM._parse_logprobs..)keyrF)rnrlr) rrrcontinuation_logprobs_listscontinuation_logprobstop_logprobs_lists is_greedytoken top_logprobs top_tokenr.r.rJrs   "zSGLangLM._parse_logprobsrGcCsb|dd|d<|dd}|durd|vrtdd|d<|dd|d<|dd|d<|S)Nrg do_sampleFzSGot `do_sample=False` and no temperature value, setting VLLM temperature to 0.0 ...skip_special_tokensspaces_between_special_tokens)rrr:debug)rGrr.r.rJrs zSGLangLM.modify_gen_kwargs)rNNrFNrrTrrNrrNrrNN)F)NFNFrr)NFF)T)"__name__ __module__ __qualname__rr9rr3rboolfloatr0rr rarrrpropertyrr*rhrrgrrrrrrm staticmethodrrr __classcell__r.r.rHrJr!s    X F a           ; #r) rloggingimportlib.utilrtypingrrrrrrr lm_eval.api.instancer lm_eval.api.modelr lm_eval.api.registryr lm_eval.models.utilsr rr lm_eval.utilsrr getLoggerrr:rr<r1rr.r.r.rJs(