o 5ti?@sddlmZddlZddlZddlZddlZddlZddlmZddl m Z m Z m Z m Z ddlmZddlmZmZeeZe dZe rgddlmZmZmZmZddlZdd lmZdd lm Z dd l!m"Z"Gd d d eddZ#dtduddZ$GdddZ%GdddZ&ddZ'    dvdwd%d&Z(Gd'd(d(Z) dxdyd.d/Z*dzd5d6Z+d{d9d:Z,d|d@dAZ- Bd}d~dFdGZ.    H  I IdddTdUZ/ Vddd\d]Z0 I V  HdddbdcZ1ddhdiZ2dxddmdnZ3dxddrdsZ4dS)) annotationsNwraps) TYPE_CHECKINGAnyLiteralTypeVar) TypedDict) maybe_warn warning_onceT)CallableIterableIteratorSequence)Image)PreTrainedTokenizerBase)PretrainedConfigc@s6eZdZUded<ded<ded<ded<d ed <d S) GenKwargsbool do_samplefloat temperatureint max_gen_toks list[str]untilr__extra_items__N)__name__ __module__ __qualname____annotations__r"r"H/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/models/utils.pyr!s  rF)totalnrccsXg}t|D]\}}||t||r|||n|kr"|Vg}q|r*|VdSdS)a Divides an iterable into chunks of specified size or based on a given function. Useful for batching Parameters: - iter: The input iterable to be divided into chunks. - n: An integer representing the size of each chunk. Default is 0. - fn: A function that takes the current index and the iterable as arguments and returns the size of the chunk. Default is None. Returns: An iterator that yields chunks of the input iterable. Example usage: ``` data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] for chunk in chunks(data, 3): print(chunk) ``` Output: ``` [1, 2, 3] [4, 5, 6] [7, 8, 9] [10] ``` N) enumerateappendlen)iterr%fnarrixr"r"r#chunks*s  r.c@s*eZdZd ddZd ddZdd d Zd S) MultiChoicereturnNonecCs ||_dSNchoices)selfr4r"r"r#__init__Qs zMultiChoice.__init__rcCs`|dD](}tt|j|dkr-td|jD] }td|qtd|dqdS)N,rzAvailable tasks to choose:z - 'z' is not in task listT)splitr(fnmatchfilterr4 eval_loggerinfo ValueError)r5valuesvaluechoicer"r"r# __contains__Us  zMultiChoice.__contains__rccs|jEdHdSr2r3r5r"r"r#__iter__^szMultiChoice.__iter__Nr0r1)r0r)r0r)rrr r6rBrDr"r"r"r#r/Ps   r/c@s*eZdZdZd ddZddZdd Zd S) Grouperz takes an array `arr` and function `fn` and returns a dictionary with keys fn(ob) for each ob in `arr` and with values `self.arr[key]` a list of all objects in `arr` satisfying `key == fn(ob)`. r0r1cs@t||_tt|}dd}||fdd}||_d|_dS)NcSs*tt}|D] }||||q|Sr2) collections defaultdictlistr')r+r*resobr"r"r#group_return_dictns z+Grouper.__init__..group_return_dictc |dSNr"r-r*r"r#u z"Grouper.__init__..)r(sizerIr&r+_grouped)r5r+r*rLr"rQr#r6is   zGrouper.__init__cCsB|jr|jSi}|jD]}dd|j|D||<q ||_|S)NcSg|]}|dqSrOr").0yr"r"r# z'Grouper.get_grouped..)rUr+keys)r5groupedkeyr"r"r# get_grouped{szGrouper.get_groupedcCsdg|j}dg|j}||jksJ|D]}t|j|||ddD]\\}}}|||<d||<q)qt|s?J|S)NFTstrict)rTr\r+zipall)r5 grouped_dictrJcovr^ind_vr"r"r# get_originals   $  zGrouper.get_originalNrE)rrr __doc__r6r_rir"r"r"r#rFbs   rFcCs&ddtjtjdd|DDS)an Undoes https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.distribute . Re-interleaves results that have been split using more_itertools.distribute: >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6]) >>> list(group_1) [1, 3, 5] >>> list(group_2) [2, 4, 6] >>> undistribute([group_1, group_2]) [1, 2, 3, 4, 5, 6] Handles non-uniform component lengths: >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7]) >>> [list(c) for c in children] [[1, 4, 7], [2, 5], [3, 6]] >>> undistribute(children) [1, 2, 3, 4, 5, 6, 7] Also handles when some iterables are empty: >>> children = distribute(5, [1, 2, 3]) >>> [list(c) for c in children] [[1], [2], [3], [], []] >>> undistribute(children) [1, 2, 3] cSsg|]}|dur|qSr2r"rXr-r"r"r#rZs z undistribute..cSsg|]}t|qSr")rIrkr"r"r#rZr[) itertoolschain from_iterable zip_longest)iterabler"r"r# undistributes rq@? on_exceptionslist[type[Exception]] max_retries int | None backoff_timerbackoff_multiplieron_exception_callback(Callable[[Exception, float], Any] | Nonecsdfdd }|S)aRetry on an LLM Provider's rate limit error with exponential backoff For example, to use for OpenAI, do the following: ``` from openai import RateLimitError # Recommend specifying max_retries to avoid infinite loops! @retry_on_specific_exceptions([RateLimitError], max_retries=3) def completion(...): # Wrap OpenAI completion function here ... ``` funcr cs"tfdd}|S)Nc s}d}dus |krIz|i|WSty>}zdur'||t||9}|d7}WYd}~nd}~wwdus |ks dSdS)NrrO)tupletimesleep)argskwargs sleep_timeattempte)ryrxr|rvrzrtr"r#wrappers  z@retry_on_specific_exceptions..decorator..wrapperr)r|rryrxrvrzrt)r|r# decorators z/retry_on_specific_exceptions..decoratorN)r|r r")rtrvrxryrzrr"rr#retry_on_specific_exceptionssrc@seZdZdZdddddfdd?ddZd@d%d&ZdAd)d*Z dBd-d.Z d/d0Z e 1dCdDd7d8Z e dEdFd:d;ZdS)GCollatora A class for reordering and batching elements of an array. This class allows for sorting an array based on a provided sorting function, grouping elements based on a grouping function, and generating batches from the sorted and grouped data. Objects of this class have the group_by attribute which determines the method for grouping the data while batching it. Three options include "gen_kwargs", "contexts", or None: If group_by == "gen_kwargs" then requests will be grouped by gen_kwargs If group_by == "contexts" then requests will be grouped by context + cont[:-1] If None then requests will just be reordered by length descending. cCs|Sr2r"rPr"r"r#rRszCollator.cCs|dSrNr"rPr"r"r#rRsNr+ Sequence[T]sort_fnCallable[[T], Any]group_fngroup_by(Literal['gen_kwargs', 'contexts'] | Noner0r1csp||_fdd|_fdd|_g|_t||_tt||_|jdkr+| dS|jdkr6| dSdS)NcrMrNr"rP)rr"r#rRrSz#Collator.__init__..crMrNr"rP)rr"r#rRrScontexts gen_kwargs) _group_by_sort_fn _group_fn_reorder_indicesr(_sizer}r&_arr_with_indices_group_by_context_group_by_index)r5r+rrrr")rrr#r6s     zCollator.__init__cC|j|j|jdd|_dS)z4Group the elements of a list based on their indices.rr*rNgrouprrrCr"r"r#r  zCollator._group_by_indexcCr)z(Group the array with indices by context.rrNrrCr"r"r#rrzCollator._group_by_contextrOr%rbatch_fn(Callable[[int, Iterable[T]], int] | None Iterator[T]ccs|jdkr$|jD]\}}||}|j|||d}|EdHq dS|jdkrE|dd|jD}|j|||d}|EdHdS||j}|j|||d}|EdHdS)a Generates and yields batches from the reordered array. The method of grouping and batching depends on the parameter `group_by`. If `group_by` is set to "gen_kwargs", it will batch the re-ordered values with same gen_kwargs for each batch. If `group_by` is "contexts", it caches the requests by context before batching. If `group_by` is neither "gen_kwargs" nor "contexts", it yields the reordered array Parameters: - n (int): The size of each batch. Defaults to 1. - batch_fn ([Callable[[int, Iterable], int]] | None): A function to determine the size of each batch. Defaults to None. Returns: Iterator: An iterator over batches of reordered elements grouped as per the `group_by` attribute. Yields: List of batched elements according to the `group_by` attribute. r)r%r*NrcSsg|] }t|dddqS)cSst|ddS)NrO)r(rPr"r"r#rR>sz1Collator.get_batched...r^)max)rXr@r"r"r#rZ=sz(Collator.get_batched..)rritems_reorder get_chunksr?)r5r%rrgr?batchr"r"r# get_batcheds*     zCollator.get_batchedreq_strtuple[str, str]cxt_toks list[int] cont_tokslogits torch.Tensor9Iterator[tuple[tuple[str, str], list[int], torch.Tensor]]c cs|jdkr[|jt||dd}t|}dkr/|jdd|D|||fVdS||dd|}t dd|Dd d i\}}}|j|t |||d d EdHdS|||fVdS) a= Retrieves cached single-token continuations and their associated arguments, updating indices as necessary. The behavior of this function varies depending on how the `group_by` attribute is set: - When `group_by` is "contexts": The function identifies single-token continuations by checking for keys that equate to [context+continuation][-1] and logs the indices for re-ordering. In this mode, this function can work in two scenarios: 1. Cache Hit - Single Match: If a single matching context-continuation pair is found in the cache, the function yields the original arguments. 2. Cache Hit - Multiple Matches: If multiple matching context-continuation pairs are found in the cache, the function expands the logits batch dimension to match the number of cache hits. It updates the original requests and continuation tokens. - When `group_by` is not set to "contexts": This method yields the original arguments, logits and continuation tokens, without checking for one-token continuations. Parameters: - req_str (tuple[str, str]): Original strings used for CachingLM. - cxt_toks (list[int]): Full context tokens used for lookup. - cont_toks (list[int]): Continuation tokens for which logits were generated. - logits (torch.Tensor [1, seq_length, vocab_size]): Logits generated by the model given context and continuation keys. Yields: - Iterator: - req_str (tuple[str, str]): strings used for CachingLM. - cont_toks (list[int]) : continuation tokens. - logits (torch.Tensor [1, seq_length, vocab_size]): The original logits (repeated cache hit times) rNrrOcss|]}|dVqdSrNr"rkr"r"r# xsz%Collator.get_cache..cSs*g|]}|d|dd|ddfqS)rrOrr"rkr"r"r#rZs*z&Collator.get_cache..raTr`) rrpopr}r(rextendexpandchunkrb) r5rrrr cache_hit cache_size multilogitsindicesr"r"r# get_cacheIs  *  zCollator.get_cache"list | tuple[tuple[int, Any], ...]rccsHt||jd}|jdkr|jdd|Ddd|DEdHdS)z Reorders the elements in the array based on the sorting function. Parameters: - arr (list | tuple[tuple[int, Any], ...]]): The array or iterable to be reordered. Yields: Iterator rrcSrV)rr"rkr"r"r#rZr[z%Collator._reorder..cSrVrWr"rkr"r"r#rZr[N)sortedrrrr)r5r+r"r"r#rs  zCollator._reordernewarrrIcCsRdg|j}dg|j}t|j|ddD] \}}|||<d||<qt|s'J|S)z Restores the original order of elements from the reordered list. Parameters: - newarr (list): The reordered array. Returns: list: The array with elements restored to their original order. NFTr`)rrbrrc)r5rrJrerfrhr"r"r#ris   zCollator.get_originalcCs|jSr2)rrCr"r"r#__len__szCollator.__len__r Iterable[T]r*!Callable[[T], Sequence[T] | dict]!Literal['gen_kwargs', 'contexts']dictc Cstt}|D]@}|dkr|t|||qztddt||D}|||WqttfyG|t|||Yqw|S)aq Groups elements of an iterable based on a provided function. The `group_by` parameter determines the method of grouping. If `group_by` is "contexts", the elements are grouped by [context + cont][:-1]. If `group_by` is "gen_kwargs", the elements are grouped based on the gen_kwargs dict. Parameters: - arr (Iterable): The iterable to be grouped. - fn (Callable): The function to determine the grouping. - values (bool): If True, returns the values of the group. Defaults to False. Returns: Iterator: An iterable of grouped elements. rcss2|]\}}|t|tjjrt|n|fVqdSr2) isinstancerGabcrr})rXr^r@r"r"r#rs   z!Collator.group..) rGrHrIr}r'rr TypeErrorAttributeError)r+r*rrJrK hashable_dictr"r"r#rs  zCollator.grouprccs`g}t|}t|D]\}}||t||r|||n|kr&|Vg}q |r.|VdSdS)a Divides an iterable into chunks of specified size or based on a given function. Useful for batching Parameters: - iter: The input iterable to be divided into chunks. - n: An integer representing the size of each chunk. Default is 0. - fn: A function that takes the current index and the iterable as arguments and returns the size of the chunk. Default is None. Returns: An iterator that yields chunks of the input iterable. Example usage: ``` data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] for chunk in chunks(data, 3): print(chunk) ``` Output: ``` [1, 2, 3] [4, 5, 6] [7, 8, 9] [10] ``` N)r}r&r'r()_iterr%r*r+r,r-r"r"r#rs  zCollator.get_chunks) r+rrrrrrrr0r1rE)rON)r%rrrr0r) rrrrrrrrr0r)r+rr0r)rrIr0rI)r)r+rr*rrrr0rr)r%rr*rr0r)rrr rjr6rrrrrrir staticmethodrrr"r"r"r#rs(   / = *r tokenizerr model_configPretrainedConfig | Noner0cCs|jr |S|jr|j|_|S|jr|j|_|S|r't|dddkr'd|_|S|jjdks3|jjdkr<|jdks:J|S| dd i|S) a This function checks if the (Hugging Face) tokenizer has a padding token and sets it if not present. Some tokenizers require special handling. Args: tokenizer: The tokenizer for which the padding token is to be handled. model_config: The configuration of the model. Default is None. Returns: The tokenizer after the padding token has been handled. Raises: AssertionError: If the tokenizer is of type RWKVWorldTokenizer or Rwkv5Tokenizer and the padding token id is not 0. model_typeNqwenz <|endoftext|>RWKVWorldTokenizerRwkv5Tokenizerr pad_tokenz<|pad|>) r unk_token unk_token_id pad_token_id eos_token eos_token_idgetattr __class__radd_special_tokens)rrr"r"r#configure_pad_tokens$ rstringstrdefault_placeholder image_token max_imagescCstd}g}||}|ddD]}||||kr$|||d7}q||kr-||q||dd|S)a  A utility function used for local multimodal models. It locates all `placeholder` string occurrences in the given input `string_` and replaces the first `max_count` instances with `replacement`, and all subsequent occurrences with the empty string. This is used to replace placeholder tags by model-specific image tokens like <|image_pad|> and to allow for only the first `max_count` images to be passed to a model if desired. :param string: The original string containing placeholders. :param default_placeholder: The placeholder text to be replaced. :param image_token: The token to replace the placeholder with. :param max_images: The maximum number of replacements to make. :return: The string with placeholders replaced. rNrrO)r9r'join)rrrrcountresultpartspartr"r"r#replace_placeholders0s      rimages list[list]cCsdd|DS)aX Takes in a list of lists of images, and returns a single list of all images in order. Used for some multimodal models like Llava-1.5 which expects this flattened-list format for its image processor. :param images: A list of lists of PIL images. :return: a list of PIL images, via concatenating all the sub-lists in order. cSsg|] }|D]}|qqSr"r")rX image_listimager"r"r#rZZsz&flatten_image_list..r")rr"r"r#flatten_image_listRsrrstr | list[str] | Noneeos str | NonercCsVt|tr |g}n|durg}n t|tstd||dur)||vr)|||S)zZEnsures that the `until` parameter is a list of stop sequences and includes the EOS token.NzAExpected `kwargs['until']` to be of type Union[str,list] but got )rrrIr>r')rrr"r"r#handle_stop_sequences]s   rrrdefault_max_gen_toksc CsZddl}||}|dg}t|ts|g}|dd|dd|dd|ddd}d d |D}t|d krFtt d |d t t t | |}|d}t|dd} |duro| dkrjdnd|d<n/dur| r| dkrtt d|d| dd|d<ndur| dkrtt d|d| d||d<||d<tdi|S)auNormalize generation kwargs for consistent handling across model backends. Model implementations may have different expectations for generation parameters. Args: gen_kwargs: Raw generation kwargs from the request. Expected keys include: - do_sample: Whether to use sampling (vs greedy decoding) - Required - until (str | list[str]): Stop sequence(s) for generation. - max_gen_toks | max_new_tokens | max_tokens | max_completion_tokens: Maximum tokens to generate - temperature: Sampling temperature - Other backend-specific kwargs default_max_gen_toks: Default max_gen_toks if not specified in gen_kwargs. Returns: A normalized dict containing: - do_sample (bool): Whether to use sampling (bool) - until: list[str]: List of stop sequences. - max_gen_toks (int): Maximum tokens to generate (int) - temperature (float): Sampling temperature (float). Note: will always be set to 0.0 if do_sample=False or do_sample is not specified. - All other kwargs passed through unchanged Notes: - Accepts `max_gen_toks` and other aliases. Priority: max_gen_toks > max_new_tokens > max_tokens > max_completion_tokens. Output always uses `max_gen_toks`. - When `do_sample=False`, temperature is set to 0.0 for greedy decoding. - When temperature is 0.0 and `do_sample` is not specified, `do_sample` is set to False. - Model backends may further modify the returned dict as needed (e.g., vLLM removes `do_sample` since it uses temperature directly). rNrrmax_new_tokens max_tokensmax_completion_tokens)rrrrcSsi|] \}}|dur||qSr2r")rXkrhr"r"r# sz(normalize_gen_kwargs..rOz"Multiple max token args provided: z_. Using first by priority (max_gen_toks > max_new_tokens > max_tokens > max_completion_tokens).rrgTFz do_sample=z` but temperature=zb; setting `temperature` to 0.0 for greedy decoding. For non-greedy decoding, set `do_sample=True`.z0. For non-greedy sampling, set temperature > 0.0r")copydeepcopygetrrIrrr(r r<rnextr)r?rr) rrrrrmax_token_aliasesprovidedrrrr"r"r#normalize_gen_kwargsmsT$            rTrOr Image.Imagewidthheight max_dimensionkeep_aspect_ratiorresample_filter min_width min_heightc CsP|j\}} |dur|dur|dur|S|} | } |durF|durF||kr)| |kr)|S|rAt|||| } t|| } t| | } nU|} |} nP|dur[||krP|S|} t| || } n;|durp| |kre|S|} t|| | } n&|durt| ||kr}|S|| kr|} t| || } n |} t|| | } t|| } t|| } || | f|S)aY Resizes a PIL Image object with flexible options. Args: image: The PIL Image object to resize. width: Target width in pixels. height: Target height in pixels. max_dimension: Maximum size for the longer dimension of the image. keep_aspect_ratio: If True (default) and both width and height are provided, the image is resized to fit within these dimensions while maintaining its aspect ratio. If False, the image is stretched to the exact width and height. resample_filter: The resampling filter to use for resizing. Defaults to Image.BICUBIC. min_width: Minimum width for the resized image. Defaults to 1. min_height: Minimum height for the resized image. Defaults to 1. Returns: The resized PIL Image object. If no resize parameters are provided or if the image already meets the criteria, the original image is returned. Order of precedence for resizing: 1. If width AND height are provided: - If keep_aspect_ratio is True: Fits image within bounds, preserving aspect ratio. - If keep_aspect_ratio is False: Resizes to exact dimensions (may distort). 2. Else if only width is provided: Calculates height proportionally. 3. Else if only height is provided: Calculates width proportionally. 4. Else if max_dimension is provided: Resizes the longest side to max_dimension and scales the other side proportionally. 5. If none of the above are provided, returns the original image. N)rTminrrresize) rrrrrrrroriginal_widthoriginal_height new_width new_heightratior"r"r# resize_imagesF )   rlefttokensr max_lengthside"Literal['left', 'middle', 'right']cCsp|dkr || dSdkr|d|Sdkr0|d}||}|d||| dStd|d)zVTruncate a token list to max_length using the given strategy (left, right, or middle).rNrightmiddlezUnknown truncation side=z.. Must be one of 'left', 'middle', or 'right'.)r>)rrr left_length right_lengthr"r"r#truncate_tokens1srr max_model_len min_gen_tokstuple[list[int], int]c Cst|}|||kr||fSd|d|d||d|d }|s7t|d|d|t||||d|fS||} |krQt|d |d | d ||| fS||} d krgtd |d|d|dt|d|d | d|d|t|| |d|fS)a Truncates input tokens and/or reduces max_gen_toks to fit within max_model_len. Strategy: 1. No truncation needed: If len(tokens) + max_gen_toks <= max_model_len, return as-is. 2. If shrink_gen_toks=False: Truncate context to fit max_model_len - max_gen_toks. 3. If shrink_gen_toks=True: a. First try reducing max_gen_toks (down to min_gen_toks) to fit the context. b. If context still doesn't fit, truncate context to reserve space for min_gen_toks. Args: tokens (list[int]): The input context tokens to potentially truncate. max_gen_toks (int): The maximum number of tokens to generate. max_model_len (int): The model's maximum context window size (prompt + generation). min_gen_toks (int): Lower bound for generation tokens. Defaults to 1. side (str): "left" | "right" | "middle". Defaults to "left". shrink_gen_toks (bool): Whether to adjust the generation tokens count to fit within the maximum length. Defaults to False. verbose (bool): Whether to log warnings when truncation or adjustments occur. Returns: tuple[list[int], int]: A tuple containing: - list[int]: The (possibly truncated) context tokens. - int: The adjusted maximum generation token count. Raises: ValueError: when max_model_len <= min_gen_toks. zContext length (z) + max_gen_toks (z) = z exceeds model's max length ()z. Truncating context from side=.)rz. Reducing max_gen_toks=z to z$ to fit within model context window.rzModel context window (z+) is too small to fit initial context len (z) + minimum generation len (z tokens to reserve min_gen_toks=z for generation.)r(r rr>) rrrrrshrink_gen_toksverbosectx_lenwarningnew_max max_ctx_lenr"r"r#maybe_truncateDs<% " r& generationstoplist[str] | str | Nonethink_end_tokencCsV|rt|tr |gn|}|D]}t|dkr||d}q|r)||d}|S)aR Post-processes the generated text by stripping stop sequences and optional thinking markers. Args: generation (str): The generated text to be processed. stop (list[str] | None): Stop sequence(s) to remove. Text is truncated at the first occurrence of any stop sequence. think_end_token (str | None): Token marking end of thinking section. If provided, returns only the text after this token (discarding thinking content). Returns: str: The processed generation - text before stop sequences and after thinking sections. rr)rrr(r9lstrip)r'r(r*termr"r"r#postprocess_generated_texts r-sequencebos_strstr | Iterable[str] | Nonecs6|durdSt|tr|Stfdd|DS)NFc3s|]}|VqdSr2) startswithrkr.r"r#rsz!has_bos_prefix..)rrr1any)r.r/r"r2r#has_bos_prefixs   r4r bool | Noneadd_boscCs$|durd|iS|durd|iSiS)Nrr")rr6r"r"r#_add_special_kwargss r7r)r%r)NrrrsN) rtrurvrwrxrryrrzr{r2)rrrrr0r)rrrrrrrr)rr)rrrrr0r)r)rrrrr0r)NNNTNrOrO)rrrrwrrwrrwrrrrwrrrrr0r)r)rrrrrrr0r)rOrFT) rrrrrrrrrrr0r)r'rr(r)r*rr0r)r.rr/r0r0r)rr5r6r5)5 __future__rrGr:rlloggingr~ functoolsrtypingrrrrtyping_extensionsr lm_eval.utilsr r getLoggerrr<r collections.abcr rrrtorchPILr transformersr transformers.configuration_utilsrrr.r/rFrqrrrrrrrrrr&r-r4r7r"r"r"r#sp        &:* ( - "  b g  J