o
    7ti                  	   @   s*  d Z ddlZddlZddlZddlZddlmZ eeZi dddddddd	d
d	dddddddddd	ddddddddddddddi dd	ddddddddddddd dd!dd"dd#dd$dd%dd&dd'dd(d	d)di d*dd+dd,dd-dd.d	d/d	d0d	d1d	d2dd3dd4d	d5dd6dd7d	d8dd9d	d:dddddd	dd;Z	d<d= Z
ed>kre
 Zejejd? Zeejd@dAZeeZW d   n1 sw   Y  ejdurddlZeejd@dAZeeZW d   n	1 sw   Y  g Zee	 D ]\ZZeevr3ee ejdur>ee ZndBdCedD dEZeejdFkrZdGej dDe ndGe ejdFkrmdGej dDe ndGe e dDdCeedHZ!ej"dDe dI Z#e$dJe dKe#  ee#dLd@dAZ%ej&e!e%dMdNdO W d   n	1 sw   Y  q%ejdFkrdPdQ eD Z'ndRdQ eD Z'ej(dFkrej(dI Z#nej"dI Z#e$dSe#  ee#dLd@dA!Z%ej&ejdFkrdGej ndTe'dUe%dVdWdX W d   dS 1 sw   Y  dS dS )Yz>
Take in a YAML, and output all "other" splits with this YAML
    N)tqdmabstract_algebrastemanatomy	astronomybusiness_ethicsotherclinical_knowledgecollege_biologycollege_chemistrycollege_computer_sciencecollege_mathematicscollege_medicinecollege_physicscomputer_securityconceptual_physicseconometricssocial_scienceselectrical_engineeringelementary_mathematicsformal_logic
humanitiesglobal_factshigh_school_biologyhigh_school_chemistryhigh_school_computer_sciencehigh_school_european_historyhigh_school_geography#high_school_government_and_politicshigh_school_macroeconomicshigh_school_mathematicshigh_school_microeconomicshigh_school_physicshigh_school_psychologyhigh_school_statisticshigh_school_us_historyhigh_school_world_historyhuman_aginghuman_sexualityinternational_lawjurisprudencelogical_fallaciesmachine_learning
management	marketingmedical_geneticsmiscellaneousmoral_disputesmoral_scenarios	nutrition
philosophy
prehistoryprofessional_accountingprofessional_lawprofessional_medicineprofessional_psychology)public_relationssecurity_studies	sociologyus_foreign_policyvirologyworld_religionsc                  C   sV   t  } | jddd | jddd | jdd d | jdd	d | jd
d	d |  S )Nz--base_yaml_pathT)requiredz--save_prefix_pathmmlu)defaultz--cot_prompt_pathz--task_prefix z--group_prefix)argparseArgumentParseradd_argument
parse_args)parser rI   X/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/mmlu/_generate_configs.pyrG   N   s   rG   __main__zutf-8)encodingzAThe following are multiple choice questions (with answers) about  _z.

rC   mmlu_)includetagtask
task_aliasdataset_namedescriptionz.yamlzSaving yaml for subset z to wT")allow_unicodedefault_stylec                 C   s   g | ]}d t j d| qS )rP   rO   )argstask_prefix.0categoryrI   rI   rJ   
<listcomp>   s    r`   c                 C   s   g | ]}d | qS )rP   rI   r]   rI   rI   rJ   r`      s    zSaving benchmark config to rA   )grouprS      F)indentdefault_flow_style))__doc__rD   loggingosyamlr   	getLogger__name__eval_loggerSUBJECTSrG   r[   pathsplitbase_yaml_pathbase_yaml_nameopenf	full_load	base_yamlcot_prompt_pathjsonloadcot_fileALL_CATEGORIESitemssubjectr_   appendrV   joinr\   replace	yaml_dictsave_prefix_pathfile_save_pathinfo	yaml_filedumpmmlu_subcategoriesgroup_prefixrI   rI   rI   rJ   <module>   sh  
	
 !"#$%&'()*+,-./01234=






$