o 5ti}p@s ddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z m Z mZddlmZddlmZmZddlmZmZdd lmZeeZeeZGd d d Z d e!e"e"fd e"fddZ#ddZ$de!d dfddZ% dde"ee"e!BeBBde dBfddZ&dS)N)Mapping)partial)Path)DictListOptionalUnion)utils)ConfigurableGroup GroupConfig)ConfigurableTaskTask)get_subtask_listc @seZdZdZ    dDdedBdeeBdBdededBddf d d Z  dEdeeBdBdedeeeffd d Z e d dZ e ddZ e ddZ e ddZe ddZ dFdefddZdeedeefddZdedefddZdedefdd Zdedefd!d"Zdedefd#d$Zdedefd%d&Zd'edefd(d)Zd'edefd*d+Zd'edefd,d-Zdefd.d/Zd0d1Zd2d3ZdGd4d5Zd6d7Z   dHd8eeBdBd9edBd:edBdefd;d<Z dGdeeBdBdefd=d>Z!d'efd?d@Z"dAefdBdCZ#dS)I TaskManagerzpTaskManager indexes all tasks from the default `lm_eval/tasks/` and an optional directory if provided. NT verbosity include_pathinclude_defaultsmetadatareturncs|dur t||_|_j||d_ttj_ tfddj D_ tfddj D_ tfddj D_ t t_dS)N)rrc"g|] }j|ddkr|qS)typegroup _task_index.0xselfJ/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/__init__.py ,"z(TaskManager.__init__..cs"g|] }j|ddvr|qS)r)task python_taskrrrrr r!/s cr)rtagrrrrr r!6r")r setup_loggingrrinitialize_tasksrsortedlistkeys _all_tasks _all_groups _all_subtasks _all_tags collections defaultdicttask_group_map)rrrrrrrr __init__s(  zTaskManager.__init__cCsn|rtjtjtdg}ng}|dur#t|tr|g}||i}|D] }||}i||}q'|S)aCreates a dictionary of tasks indexes. :param include_path: Union[str, List] = None An additional path to be searched for tasks recursively. Can provide more than one such path as a list. :param include_defaults: bool = True If set to false, default tasks (those in lm_eval/tasks/) are not indexed. return Dictionary of task names as key and task metadata /N) ospathdirnameabspath__file__ isinstancestrextend_get_task_and_group)rrr all_paths task_indextask_dirtasksrrr r';s   zTaskManager.initialize_taskscC|jSN)r+rrrr all_tasksZzTaskManager.all_taskscCrArB)r,rrrr all_groups^rDzTaskManager.all_groupscCrArB)r-rrrr all_subtasksbrDzTaskManager.all_subtaskscCrArB)r.rrrr all_tagsfrDzTaskManager.all_tagscCrArBrrrrr r>jrDzTaskManager.task_indexcCsddlm}dd}|}ddg|_g}|jD]}|j|d} | dkr'd } n|| } ||| gq||_|} d g| _d d |jD| _|} gd | _g} |jD]R} |j| d} d}| dkrt j | dd}d|vrr|d}n d|vr| ddd|d}t j |dd}d|vr|d}| dkrd } n|| } | | | |gqS| | _d}|r|| d7}|r|| d7}|r|| d7}|S)Nr)MarkdownTableWritercSsd|vr d|ddS|S)Nzlm_eval/tasks/)split)r5rrr sanitize_pathssz1TaskManager.list_all_tasks..sanitize_pathGroupConfig Location yaml_pathrIz---TagcSsg|]}|gqSrr)rtrrr r!sz.TaskManager.list_all_tasks..)r rMz Output Typesimplemode output_typeincluder3 z ) pytablewriterrHheadersrEr>append value_matrixrGrFr load_yaml_configrJdumps)r list_groups list_tags list_subtasksrHrK group_table gt_valuesgr5 tag_table subtask_table st_valuesrPrUconfigrinclude_configresultrrr list_all_tasksnsV      zTaskManager.list_all_tasks task_listcCst||jSrB)r pattern_matchrC)rrkrrr match_tasksszTaskManager.match_tasksnamecCs||jvrdSdS)NTF)rCrrnrrr _name_is_registereds zTaskManager._name_is_registeredcC$||r|j|ddkrdSdS)Nrr#TFrpr>rorrr _name_is_taskzTaskManager._name_is_taskcCrq)Nrr%TFrrrorrr _name_is_tagrtzTaskManager._name_is_tagcCrq)NrrTFrrrorrr _name_is_group zTaskManager._name_is_groupcCrq)Nrr$TFrrrorrr _name_is_python_taskrwz TaskManager._name_is_python_taskrgcCd|vr t|dtr dSdSNr#TFr9r:rrgrrr _config_is_taskzTaskManager._config_is_taskcCryrz)r9r)r|rrr _config_is_groupr~zTaskManager._config_is_groupcCsd|vrdSdS)NclassTFrr|rrr _config_is_python_tasksz"TaskManager._config_is_python_taskcCs||jvrt|j|dS)NrN)r> ValueErrorrorrr _get_yaml_path zTaskManager._get_yaml_pathcCs2||jvrt||}|dkriStj|ddS)NrIfullrS)r>rrr r\)rrnrNrrr _get_configs  zTaskManager._get_configcCs||rt|j|dS)Nr#)rsrr>rorrr _get_tasklistrzTaskManager._get_tasklistcCs0d|vrd|vr|dur|d|krd|d<|S)N group_aliasrr)rrgrrrr _process_aliass zTaskManager._process_aliascCs$t|dd}|rdt|jvSdS)Nr2rgF)getattrinspect signature parameters)rcls constructorrrr _class_has_config_in_constructors z,TaskManager._class_has_config_in_constructorname_or_config parent_name update_configc sfdd}dtdttttfffdd } ddtdtdtttffdd }t|tr|dur8d |i|}nW|sB|rM|}|||d S |}|d krh|} || \} }|| \} }n' |rt j t|trw|ndd } tt jt| t|St||dd} t|tr.|r"|d |duri||}rˆ} || |\} }|| \} }nl r߈ }t j |d } tt jt| t|Sr} |durtttfddj|} | dkr d| j|i| |}n|}||d S||\} }|| \} }t j | |d} | tt jt| t|iS)Ncsd|vritjdd|didd|}|r;|dr)|d|d}n|d}t|tr7||j_||iSj durL| dij B|d<n| di|d<t|d}||iS)NrVr)rN yaml_configrTrrgr) r r\poprrr9r rgr#rget)rgr# task_objectrrr _load_task s,      z>TaskManager._load_individual_task_or_group.._load_taskrgrcsvjdur|dijB|d<t|d}g}|jdD]}t|tr1|r1||q| |q||fS)Nrrr#) rrr rgr9r:rur;rrZ)rg group_name subtask_listr#rrr "_get_group_and_subtask_from_config$s   zVTaskManager._load_individual_task_or_group.._get_group_and_subtask_from_configrcSsL|dur i||}dd|D}t|sd}dd|D}||fS)NcSsi|] \}}|tvr||qSrGROUP_ONLY_KEYSrkvrrr 7z]TaskManager._load_individual_task_or_group.._process_group_config..cSsi|] \}}|tvr||qSrrrrrr r=s)itemsbool)rgr_update_config group_configrrr _process_group_config2s zITaskManager._load_individual_task_or_group.._process_group_configr#r#rI)r)rr#rcs |SrB) startswith)r)rnrr  z.r-)rrrB)dicttupler r)r:r9rsrxrrrur_load_individual_task_or_groupr/ChainMapmapreversedr}rrvrplenfilterr1rZ)rrrrrrr task_configrrrfnbase_task_config num_duplicater)rnrr rs                        z*TaskManager._load_individual_task_or_groupcs0t|tr|g}ttjtfdd|}|S)zLoads a dictionary of task objects from a list :param task_list: Union[str, list] = None Single string or list of string of task names to be loaded :return Dictionary of task objects cs |SrBrrrrr rrz0TaskManager.load_task_or_group..)r9r:rr/rr)rrkall_loaded_tasksrrr load_task_or_groups  zTaskManager.load_task_or_groupcCs ||SrBrr|rrr load_configs zTaskManager.load_configr?c sVdd}d}ddgt}t|D]\}}}fdd|D|dd<|||D]x}|d rtj||} tj | d d } | | r\| d } d | d|| <|| | ||q/| | rldd| d|| d<q/| | r| d } | |vrt d| d|| dd| q/d | d|| <|| | ||q/t d|d|dq/q|S)aCreates a dictionary of tasks index with the following metadata, - `type`, that can be either `task`, `python_task`, `group` or `tags`. `task` refer to regular task configs, `python_task` are special yaml files that only consists of `task` and `class` parameters. `group` are group configs. `tags` are labels that can be assigned to tasks to assist in sorting and calling tasks of certain themes. - `yaml_path`, path to the yaml file. If the entry is a `group` that was configured through a task config, the yaml_path will be -1 and all subtasks will be listed in `task` (see below) - `task`, reserved for entries with `type` as `group`. This will list all subtasks. When a group config is created (as opposed to task config having `group` parameter set), this will be set to -1 to avoid recursive indexing. The whole list of subtasks will be loaded at evaluation. :param task_dir: str A directory to check for tasks :return Dictionary of task names as key and task metadata cSsd|vr@|d}t|tr|g}|D]/}||vr"d|gdd||<q||ddkr6td|ddS||d|qdSdS)Nr%rIrr#rNrz The tag 'zl' is already registered as a group, this tag will not be registered. This may affect tasks you want to call.r#)r9r: eval_loggerinforZ)rgr#tasks_and_groups print_info attr_listr%rrr _populate_tags_and_groupss$   zBTaskManager._get_task_and_group.._populate_tags_and_groupsT __pycache__z.ipynb_checkpointscsg|]}|vr|qSrr)rd ignore_dirsrr r!sz3TaskManager._get_task_and_group..Nz.yamlrRrSr#r$)rrNrrIrzDuplicate task name 'z"' found. Already registered from: rNz. Skipping duplicate from: zFile z in z could not be loaded)r/r0r4walksortendswithr5joinr r\rrr}rwarningdebug) rr?rrrrootdirs file_listfrNrgr#rrr r<s`       7zTaskManager._get_task_and_group)NNTN)NT)TTTrB)NNN)$__name__ __module__ __qualname____doc__r:r)rrr2r'propertyrCrErFrGr>rjrmrprsrurvrxr}rrrrrrrrrrrr<rrrr rs  "         B     rrrcCs8d|vr|dSd|vrdjdi|Sdjdi|S)Nr# dataset_namez{dataset_path}_{dataset_name}z{dataset_path}r)format)rrrr get_task_name_from_config+s rcCs.t|dr |jdSt|dr|jSt|jS)Nrgr#EVAL_HARNESS_NAME)hasattr_configrrr)rrrr get_task_name_from_object4s  r task_dictcstgD] \}}|qfddDfddD}tdkr8tdtd|dd S) ahelper function solely used in validating get_task_dict output. Takes the output of lm_eval.evaluator_utils.get_subtask_list and returns a list of all leaf subtasks contained within, and errors if any such leaf subtasks are "oversubscribed" to several disjoint groups. csh|] }|dkr|qS))count)r task_name) subtask_namesrr Krz$_check_duplicates..cs*g|]}tt|dkr|qS)r)rset intersection)rr)duplicate_tasksrrr r!Ps z%_check_duplicates..rzjFound 1 or more tasks while trying to call get_task_dict() that were members of more than 1 called group: z. Offending groups: zW. Please call groups which overlap their constituent tasks in separate evaluation runs.N)rr;r*rrr))rkeyvaluecompeting_groupsr)rrrr _check_duplicatesAs    rtask_name_list task_managercCsi}i}i}t|tr|g}nt|tr"tdd|Ds!tdn tdt|ddd|D}dd|D}t|dkrL|d urGt}||}|D]!}t|t r`i||j |d }qNt|t roi|t ||i}qNt |t |sti|||}tt|d tfd d } td|D]\} } t| tr2td| jt| t r'tt| } t| tr| D]R\} }t| trtd| j|D]\}}t|tr| ||ddqt|d|qqt| trt|tr| | |ddqtd| d|qqt| d| qt| d| qt| trFt| trF| | |ddqt| d| q|S)aCreates a dictionary of task objects from either a name of task, config, or prepared Task object. :param task_name_list: List[Union[str, Dict, Task]] Name of model or LM object, see lm_eval.models.get_model :param task_manager: TaskManager = None A TaskManager object that stores indexed tasks. If not set, task_manager will load one. This should be set by the user if there are additional paths that want to be included via `include_path` :return Dictionary of task objects cSsg|] }t|tttfqSr)r9r:rr rr#rrr r!usz!get_task_dict..zfExpected all list items to be of types 'str', 'dict', or 'Task', but at least one entry did not match.z(Expected a 'str' or 'list' but received .cSsg|] }t|tr|qSrr{rrrr r!~scSsg|] }t|ts|qSrr{rrrr r!s  rNrindentcSsn|j|d}t|}ttj}z||}Wn ty"|}Ynwd|}t|d|d|ddS)NrN zTask: z ())r>rr8parent relative_torrr)rrrrNlm_eval_tasks_path display_pathpadrrr pretty_print_tasks   z(get_task_dict..pretty_print_taskzSelected tasks:zGroup: z Subgroup: )rz: rr)r9r:r)all TypeErrorrrrrrrr rrr* isdisjointrrrintrrrr rnextiterr )rrtask_name_from_string_dicttask_name_from_config_dicttask_name_from_object_dictstring_task_name_listothers_task_name_list task_elementfinal_task_dictrrr first_keysubgrouprrconfigurable_taskrrr get_task_dict\s                r rB)'r/rloggingr4collections.abcr functoolsrpathlibrtypingrrrrlm_evalr lm_eval.api.groupr r lm_eval.api.taskr r lm_eval.evaluator_utilsrr)to_dictr*r getLoggerrrrrr:rrrr rrrr s8