o
    7ti                  
   @   s~  d Z ddlZddlZddlZddlmZ ddlmZ ee	Z
ee jZed Zed Zed Zg dZd	Zd
ZdefddZdededefddZdededefddZdedefddZdededdfddZdededededef
ddZdededefd d!Zded"ee defd#d$Zd%ee defd&d'Zd(d) Z e	d*kr=e  Z!e!j"j#Z$eeZ%eeZ&e!j&re'e!j&Z(d+d, e&D Z&g Z)e&D ]fZ*e*d- Z+e%, D ]\Z-Z.e!j/d.e+ d/e- d0 Z0ee0ee$e*e-e. qg Z1eD ]!Z.e12d.e+ d/e.  e!j/d1e+ d/e. d0 Z3ee3ee*e. qd.e+ Z4e!j/d1e+ d0 Z5ee5ee*e1 e)2e4 qe!j/d2 Z6ee6ee) dS dS )3z9Generate MMMLU YAML configs for every locale and subject.    N)Path)Iterabledefaultzsubjects.jsonzlanguages.json)stemothersocial_sciences
humanitiesznaggregate_metric_list:
  - metric: acc
    weight_by_size: True
  - metric: acc_norm
    weight_by_size: True
z@aggregate_metric_list:
  - metric: acc
    weight_by_size: True
pathc                 C   s:   | j dd}t|W  d    S 1 sw   Y  d S )Nutf-8encoding)openjsonload)r	   handle r   `/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/openai-mmmlu/_generate_configs.py	load_json   s   $r   subjectdisplay_namereturnc                 C   s"   d | d}d| d| dS )N _zAThe following are multiple choice questions (with answers) about  (z).

)joinsplit)r   r   subject_textr   r   r   description_for$   s
   r   c                 C   s   |  dd d| dS )Nr   r   r   )replace)r   r   r   r   r   subject_alias,   s   r!   valuec                 C   s(   |  dd dd dd}d| dS )N\z\\"z\"
z\nr   )r"   escapedr   r   r   quote0   s
   
r'   contentc                 C   s&   | j jddd | j|d dd d S )NT)parentsexist_okr%   r
   r   )parentmkdir
write_text)r	   r(   r   r   r   
write_file9   s   r.   	base_yamllanguagecategoryc                 C   s   |d }|d }|d }dt |  dt | d| dt d| d	| d
 dt d| d	|  dt t|| dt t|| g}d|S )Ndataset_namer   slugz"include": z"dataset_name": z&process_docs: !function utils.process_z"tag": mmmlu_r   _tasksz"task": z"task_alias": z"description": r%   )r'   r!   r   r   )r/   r0   r   r1   r2   r   r3   linesr   r   r   subject_yaml>   s   
	r7   c                 C   sj   | d }| d }| d| dd }d| d| dt| dd| d| d	g}d
|tddg S )Nr3   r   r   r   group: mmmlu_group_alias: task:z
  - mmmlu_r5   r%   	metadata:  version: 1)r    r'   r   CATEGORY_METRICS)r0   r1   r3   r   labelheaderr   r   r   category_yamlN   s   r@   
categoriesc                 C   sN   | d }d| dt | d  dg}|dd |D  d|td	d
g S )Nr3   r8   r9   r   r:   c                 s       | ]}d | V  qdS z  - Nr   ).0r1   r   r   r   	<genexpr>b       z&language_group_yaml.<locals>.<genexpr>r%   r;   r<   )r'   extendr   LANGUAGE_METRICS)r0   rA   r3   r?   r   r   r   language_group_yaml[   s   rI   language_groupsc                 C   s0   g d}| dd | D  d|tddg S )N)zgroup: mmmluzgroup_alias: "MMMLU"r:   c                 s   rB   rC   r   )rD   groupr   r   r   rE   l   rF   z$master_group_yaml.<locals>.<genexpr>r%   r;   r<   )rG   r   rH   )rJ   r?   r   r   r   master_group_yamlf   s   rL   c                  C   sJ   t  } | jdtd tdd | jdttdd | jddd d	d
 |  S )Nz--base_yaml_path_default_template_yamlz Path to the shared template YAML)r   typehelpz--output_dirz'Directory to place generated YAML filesz--languages*z-Optional subset of language slugs to generate)nargsr   rO   )argparseArgumentParseradd_argumentDEFAULT_OUTPUT_DIRr   
parse_args)parserr   r   r   rV   p   s(   rV   __main__c                 C   s   g | ]
}|d  t v r|qS )r3   )	requested)rD   langr   r   r   
<listcomp>   s    r[   r3   r4   r   z.yaml_mmmlu_z_mmmlu.yaml)7__doc__rR   r   loggingpathlibr   typingr   	getLogger__name__LOGGER__file__resolver+   
SCRIPT_DIRrU   SUBJECTS_PATHLANGUAGES_PATHCATEGORY_ORDERrH   r=   r   strr   r!   r'   r.   dictr7   r@   rI   rL   rV   argsbase_yaml_pathnamer/   subjects	languagessetrY   rJ   r0   r3   itemsr   r1   
output_dir	file_pathcategory_namesappendcategory_pathlanguage_grouplanguage_pathmaster_pathr   r   r   r   <module>   sd   
	




