o 5tixZ @s`ddlZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z ddl m Z ddlmZeeZGdddZded eefd d Zd"d d Zd#ddZdedBd edBfddZ d$deded eeeffddZdeed eeeeeeeffddZ   d%d eeeedffddZedej d ej fddZ!edee"fd d!Z#dS)&N)ConfigurableGroup)aggregate_subtask_metricsmeanpooled_sample_stderrstderr_for_metric)Task)positional_deprecatedc@sReZdZdZ         dddZedefddZddd d Zd d Z dS) TaskOutputa Wrapper class for Task outputs.It contains various attributes and methods to manage and calculate metrics for the task. Attributes: task (object): The task object. task_name (str): The name of the task. task_config (dict): The configuration of the task. version (str): The version of the task. group_name (str): The name of the task group. n_shot (int): The number of shots for the task. task_alias (str): The alias of the task. group_alias (str): The alias of the task group. is_group (bool): Indicates if the task is a group. logged_samples (list): The list of logged samples. sample_len (int): The length of the samples. sample_metrics (defaultdict): The dictionary of samples' metrics. agg_metrics (defaultdict): The dictionary of aggregate metrics. Methods: from_taskdict(cls, task_name: str, task): Creates a TaskOutput instance from a task dictionary. calculate_aggregate_metric(bootstrap_iters=100000) -> None: Calculates the aggregate metrics for the task. Nc Cs^||_||_||_||_||_||_||_||_| |_g|_ d|_ t t |_t t |_dSN)task task_config task_name group_nameversionn_shot task_alias group_aliasis_grouplogged_samples sample_len collections defaultdictlistsample_metrics agg_metrics) selfr r r rrrrrrrK/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/evaluator_utils.py__init__0s  zTaskOutput.__init__r c Cst|tr |\}}nd}|sd}|||||dS|j}t|}|d}dkr4|didd}|d}|d} ||||||||| dS) NT)r r rr num_fewshotrmetadataaliasr)r r r rrrrr) isinstancetupleVERSIONdict dump_configget) clsr r rrrr rrrrrr from_taskdictJs0     zTaskOutput.from_taskdict順returnc Cs|jD]`\\}}}z |j|}Wn tyt}Ynw|d|}|||j|<t||_t |t r^t ||dvrCt |dn|d}|rSt|dkrS||nd|j|d|<qt d|d dS) N,)bleuchrfterd)metricbootstrap_itersN/A_stderr,zReceived bootstrap_iters 'zD' but expected an integer. Set to 0 to turn off stderr calculations.)ritemsr aggregationKeyErrorrrlenrr"intrmin ValueError)rr2r1 filter_keyr6agg_fn metric_key stderr_fnrrrcalculate_aggregate_metrichs,     z%TaskOutput.calculate_aggregate_metricc Cs6d|jd|jd|jd|jd|jd|jd S)NzTaskOutput(task_name=z , group_name=z , version=z , n_shot=z , task_alias=z, group_alias=))r rrrrr)rrrr__repr__s zTaskOutput.__repr__) NNNNNNNNN)r*r+N) __name__ __module__ __qualname____doc__r classmethodstrr)rArCrrrrr s    r task_dictr+cCsNg}|D]\}}t|trt|}||qt||}||q|Sr )r6r"r% get_task_listextendr r)append)rKoutputsr task_obj_outputs task_outputrrrrLs    rLc s i}|D]c\}}t|tr|j}n|}t|tr@t||dd}|r9||fgfdd|Di||}qt|trI|j}nt|t rQ|j }|dur^||fgq||fg |qdkri}|D] \} } | \}| ||<qt|}|S)Nr3) task_rootdepthcs g|] \}}|dkr|qS)r3r).0_task_depthrTrr s  z$get_subtask_list..r) r6r"rrr%get_subtask_list setdefaultrMkeysrr rN) rKrSrT subtask_list group_objrPr _subtask_listgroup_or_task_name group_key task_listrrXrrZs@       rZc Csb|jD]+}|jdkr.td|d|jd|jdd||jd tdt|qdS) Nr3zTask: z ; document z.; context prompt (starting on next line): rz` (end of prompt on previous line) target string or answer choice index (starting on next line): z! (end of target on previous line)z Request: ) instancesdoc_id eval_loggerinfoargs doc_to_targetdocrJ)r instrrrprint_writeouts   rklimitcCs4|dur|dkrttt|j|nt|}|S)Ng?)r:mathceilr9 eval_docs)r rlrrrget_sample_sizes&rpresultscCsdd}tt}tt}||}|D]\}}|dkr$d|dnd} t|tr5|j} d} ||}nt|trF|} t|trD|j } d} ||  || <| r]|j d urY|j } n|j } nd || vrj|| d } n| } | | || d <d || vr||  d | rd|| vr|dkrd|dnd} ||  || <| | || d <d || vr||  d t|tr|d 7}|d 7}t||||\}}i||}i||}|d 8}|d 8}q||fS) a @param task_dict: Dictionary representing the group hierarchy of tasks. Each key is a group name and its value is a list of task names. @param results: Dictionary containing the results of each task. Each key is a group name and its value is a dictionary of task results. @param task_depth: The indentation level for printing the task hierarchy. Default is 0. @param group_depth: The indentation level for printing the group hierarchy. Default is 0. @return: A tuple of two dictionaries: results_agg and groups_agg. results_agg contains aggregated results for each task, and groups_agg contains aggregated results for each group. Prepares the task hierarchy and aggregates the results for each task and group recursively for printing. cSstt|dddS)z Helper utility. Sorts the task dict at the current level of the hierarchy based on alphabetized task name. Required so that we end up sorting within each sub-header correctly. cSs t|dtr |djS|dSNr)r"rr)itemrrrs  z>prepare_print_tasks.._sort_task_dict..)key)r%sortedr6)rKrrr_sort_task_dicts z,prepare_print_tasks.._sort_task_dictr z- TFNr!samplesr3)rrr%r6r"rrrJrr copyrgrouppopprepare_print_tasks)rKrq task_depth group_depthrwtask_agg group_aggtask_or_group_nametask_or_group_obj tab_stringnamefrom_configurable_groupr!group_tab_string _task_agg _group_aggrrrr~s`            r~ eval_taskscCsbtt}tt}tt}tt}tt}tt}|D]}d|j}vr3|d||jd<n|j||jd<|j} rO| |vrO|j} rO| || d<|j ||j<|j||j<|j ||j<|j ||j<|j ||j<|jD]3\\} } } | d| }|j|||j|<|j||jd<|j| d| ||j| d| <qtq ||||||fS)a( @param eval_tasks: list(TaskOutput). @return: A tuple containing the consolidated results, samples, configs, versions, and num_fewshot. Consolidates the results of multiple evaluation tasks into a single structure. The method iterates over each evaluation instance and extracts relevant information to create the consolidated results structure. The consolidated results structure has the following properties: - results: A defaultdict with task names as keys and dictionaries as values. Each dictionary contains metric/filter pairs as keys and corresponding metric values as values. The "alias" key is used to store task aliases specified in the task configuration. - samples: A defaultdict with task names as keys and lists of log samples as values. - configs: A defaultdict with task names as keys and task configurations as values. - versions: A defaultdict with task names as keys and task versions as values. - num_fewshot: A defaultdict with task names as keys and number of few-shot samples as values. - higher_is_better: A defaultdict with task names as keys and indicators of whether higher values are better for each metric as values. The method then returns the consolidated results, samples, configs, versions, and num_fewshot as a tuple. rr!r,rzr5)rrr%rr:r r rrrrrr higher_is_betterrr6rr)rrqrzrconfigsversionsrrRr rrr1r=r6r?rrrconsolidate_results8s8             rFc s|duri}|dur i}|D]O\}}t|tr!|j}|j}nd}t|tr5|r4||g|jqt |||||\}}} |rQ||g | |g|dus[|ddurbd|d<qd|vrj|d} |t |dB}| |} t fdd| D} | D]ddfdd | D} fd d | D}fd d | D}| D]V}|d D]O}d|d |gkrq|ddkrt}nt|dr|d}n td|dd|d|| ||d|<d|vrd|<qt|||<qqt||d<| dd}|dur| dd||<q| D]=}dd |D}dd |D}|r^t|t|kr^|D]}||d|dd}||dqDq"q|||fS)a (Recursively) calculates groups' aggregated metrics and updates the results and versions dictionaries with this info. @return: a tuple [results, versions, show_group_table, task_aggregation_list] with formats described below: - results: A defaultdict with task names (and, after this function is called, group names of groups that perform aggregation) as keys, and dictionaries with "alias" and metric,filter_name pairs as keys. - versions: A defaultdict with task names (and, after this function is called, group names of groups that perform aggregation) as keys, and float values representing the task or group's version if a version is specified. (defaulting to None). - show_group_table: a boolean which is true if there exists a group that requires printing of its aggregated scores in a group table. - task_aggregation_list: a defaultdict listing the subtasks to average over to produce a given group's end metric. The method then returns the updated results, versions, show_group_table, and task_aggregation_list as a tuple. In the top-level invocation of this function, task_aggregation_list is ignored. Naggregate_metric_listrxcs2h|]}|D] }d|vr |dvr |q qS)_stderr)r r!rz)r\)rUr ru)rqrr s z,consolidate_group_results..r5r,cs$g|]}|vr|qSrrrUr r1rqrrrY   z-consolidate_group_results..cs$g|]}|vr|qSrrr)rqstderrrrrYrcs$g|]}|vr|dqS)rzrrrrrrYr filter_listr1r7rzgCurrently, only 'mean' is supported for automatically aggregating scores across groups' subtasks. Got 'z ' for group ''weight_by_sizer4rzr rcSs"g|] }d|vr|ds|qS)r, score_stderr startswithrUrurrrrYs cSsg|] }|dr|qS)score,rrrrrrYs  rz score_stderr,)r6r"rconfigrrr[rNr consolidate_group_resultsrMr'boolrjoinsplitrcallabler<rsumr\r9r}replace)rqrrKrSshow_group_tabletask_aggregation_list group_or_taskgroup_or_task_info group_config_task_aggregation_listagg_metric_listrb metric_listmetricsstderrssizes metric_config filter_name aggregate_fngroup_metadatar task_metrics score_metrics score_metric stderr_keyr)r1rqrrrvs                      r start_pathcCsV|}d}t|D]}|ddr|S|j}q td|dd|)z Search upward in the directory tree to a maximum of three layers to find and return the package root (containing the 'tests' folder) testsztest_version_stable.pyz#Unable to find package root within z upwardszof )resolverangeexistsparentFileNotFoundError)rcur_path max_layers_rrrfind_test_roots  rrbcCsrddl}tttd}d|}|dd|d|g}tjt || |}|r7t d|d |dS) zE Find the package root and run the tests for the given tasks rN)rz or z/tests/test_version_stable.pyz --rootdir=z-kz'Not all tests for the specified tasks (z ) ran successfully! Error code: ) pytestrpathlibPath__file__rsyspathrNrJmainr<)rbr package_root task_stringrgpytest_return_valrrrrun_task_tests%s  rrrrD)rr)NFN)$rloggingrmrrlm_eval.api.grouprlm_eval.api.metricsrrrrlm_eval.api.taskr lm_eval.utilsr getLoggerrErer r%rrLrZrkr:rpr#r~rrrrrrJrrrrrsP    x /    \ B