o
    7ti                  
   @   s   d Z ddlZddlZddlZddlZddlmZ eeZi dddddddd	d
d	dddddddddd	ddddddddddddddi dd	ddddddddddddd dd!dd"dd#dd$dd%dd&dd'dd(d	d)di d*dd+dd,dd-dd.d	d/d	d0d	d1d	d2dd3dd4d	d5dd6dd7d	d8dd9d	d:dddddd	dd;Z	d<gZ
d=d> Zed?kre ZejejZeejd@dAdBZeeZW d   n1 sw   Y  ejdurddlZeejdAdBZeeZW d   n	1 sw   Y  e
D ]Zee	 D ]\ZZejdur8ee ZndCdDedE dFZeej rTdGej  e dEe ndGe dEe ej!rjdGej! e dEe ndGe dEe e"dEdDee dEe dHZ#ejej$e dEe dIZ%ee%dJdAdBZ&ej'e#e&dKdLdM W d   n	1 sw   Y  e(dNe dDe dOe%  q)q!ej rejej)ej dI Z%e(dPe%  ee%dJdAdBZ&ej'e#e&dQdRdS W d   dS 1 sw   Y  dS dS dS )Tz>
Take in a YAML, and output all "other" splits with this YAML
    N)tqdmabstract_algebrastemanatomy	astronomybusiness_ethicsotherclinical_knowledgecollege_biologycollege_chemistrycollege_computer_sciencecollege_mathematicscollege_medicinecollege_physicscomputer_securityconceptual_physicseconometricssocial_scienceselectrical_engineeringelementary_mathematicsformal_logic
humanitiesglobal_factshigh_school_biologyhigh_school_chemistryhigh_school_computer_sciencehigh_school_european_historyhigh_school_geography#high_school_government_and_politicshigh_school_macroeconomicshigh_school_mathematicshigh_school_microeconomicshigh_school_physicshigh_school_psychologyhigh_school_statisticshigh_school_us_historyhigh_school_world_historyhuman_aginghuman_sexualityinternational_lawjurisprudencelogical_fallaciesmachine_learning
management	marketingmedical_geneticsmiscellaneousmoral_disputesmoral_scenarios	nutrition
philosophy
prehistoryprofessional_accountingprofessional_lawprofessional_medicineprofessional_psychology)public_relationssecurity_studies	sociologyus_foreign_policyvirologyworld_religionsquestion_and_answerc                  C   s\   t jdd} | jdddd | jddd	 | jd
dd	 | jdd d	 | jddd	 |  S )Nz<Generate configuration YAML files for LM Evaluation Harness.)descriptionz--base_yaml_pathTz)Path to the base YAML configuration file.)requiredhelpz
--save_dirzN/data/local/cat/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer)defaultz--task_prefix z--cot_prompt_pathz--group_prefix)argparseArgumentParseradd_argument
parse_args)parser rK   O/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/mmlusr/config.pyrI   O   s    rI   __main__rzutf-8)encodingzAThe following are multiple choice questions (with answers) about  _z.

mmlusr_)includetagtask
task_aliasrA   dataset_namez.yamlwT")allow_unicodedefault_stylezSaved YAML for z to zSaving benchmark config to    F)indentdefault_flow_style)*__doc__rF   loggingosyamlr   	getLogger__name__eval_loggerSUBJECTSGROUPSrI   argspathbasenamebase_yaml_pathbase_yaml_nameopenf	full_load	base_yamlcot_prompt_pathjsonloadcot_filegroupitemssubjectcategoryrA   joinsplitgroup_prefixtask_prefixreplace	yaml_dictsave_dirfile_save_path	yaml_filedumpinfosave_prefix_pathrK   rK   rK   rL   <module>   sD   
	
 !"#$%&'()*+,-./01234<


 $)