o
    7ti[                     @   s   d dl Z d dlZd dlZdedefddZdejdejfddZdejdejfd	d
ZdejdejfddZdejdejfddZ	dejdejfddZ
dejdejfddZdd Zdd Zdd ZdS )    Nstringreturnc                 C   s   t | d S )Nzutf-8)hashlibsha256encode	hexdigest)r    r   X/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/metabench/process_docs.pyhash_string   s   r
   datasetc                 C      dd }|  |S )Nc                 S   s   d}t ddD ]R}| d|  }| d|  | d|  }| d|  | d|  d |}| d|  d | }| d|  | d	|  | d
| d| d}q|| d< t| d | d< | d | S )N       arc_question_shot_arc_answerKey_shot_arc_choices_shot_labeltextarc_idx_shot_
Question: 	
Answer: 

twentyfive_shot_prepromptquestionoriginal_hashalltwentyfiveshot_longprompt)rangepopindexr
   )doclong_promptshotr   
answer_lab
answer_idxanswerr   r   r	   _subprocess   s   
z process_arc.<locals>._subprocessmapr   r&   r   r   r	   process_arc   s   
r*   c                 C   r   )Nc                 S   s   d}t ddD ]3}| d|  }| d|  | d|  }| d|  | d|  | d| d| d	}qt| d
 | d< || d< | d | S )Nr   r      gsm8k_prompt_shot_gsm8k_answer_shot_gsm8k_idx_shot_r   r   r   r   r   five_shot_prepromptallfiveshot_longprompt)r   r   r
   )r    r!   r"   r   r%   r   r   r	   r&   "   s   
z"process_gsm8k.<locals>._subprocessr'   r)   r   r   r	   process_gsm8k!   s   
r1   c                    s   dd   fdd}|  |S )Nc                 S   s2   |   } | dd} tdd| } | dd} | S )Nz [title]z. z\[.*?\]r   z   )stripreplaceresub)r   r   r   r	   process_txt5   s
   z&process_hellaswag.<locals>.process_txtc                    s  | d d | d    }| d | d | d  | d d | | d<  fdd	| d
 D | d< t| d | d< | d | d
 d}tddD ]}| d|  d | d|     }| d|  | d|  | d|   | d|  d | } | d|  t| d|   }| d|  | d|  | d|  | | d| d}| d|  | d|  | d|  | d|  qMt| d | d< || d< | d | S )Nctx_ar2   ctx_bctxactivity_labelz: queryc                    s   g | ]} |qS r   r   ).0endingr7   r   r	   
<listcomp>C   s    z:process_hellaswag.<locals>._preprocess.<locals>.<listcomp>endingschoicesr   goldr   r      hellaswag_ctx_a_shot_hellaswag_ctx_b_shot_hellaswag_ctx_shot_hellaswag_activity_labels_shot_hellaswag_endings_shot_hellaswag_label_shot_r   hellaswag_ind_shot_hellaswag_source_id_shot_hellaswag_split_shot_hellaswag_split_type_shot_r   ten_shot_prepromptalltenshot_longprompt)
capitalizer   intr   r
   )r    r:   r!   r"   r   r>   r?   r   r	   _preprocess=   sP   





z&process_hellaswag.<locals>._preprocessr'   )r   rS   r   r?   r	   process_hellaswag4   s   
-rT   c                 C   r   )Nc           
      S   s6  g d}dd | d d d}tddD ]p}| d	|   }| d	|  |t| d
|   }| d|  d }| d|  d }| d|  d }| d|  d }	| d|  | d
|  | d|  | | d| d| d| d|	 d| d}qt| d | d< || d< | d | S )N)ABCDzAThe following are multiple choice questions (with answers) about r2   subject_z.

r   r+   mmlu_question_shot_mmlu_answers_shot_mmlu_choices_shot_r         mmlu_ind_shot_z
A. z
B. z
C. z
D. r   r   r   r   r/   r0   )joinsplitr   r3   r   rR   r
   )
r    rB   r!   r"   r   r%   choice_Achoice_Bchoice_Cchoice_Dr   r   r	   r&   n   s$   .
z!process_mmlu.<locals>._subprocessr'   r)   r   r   r	   process_mmlum   s   
rg   c                 C   r   )Nc                 S   s   t | d | d< | S )Nr   r   )r
   )r    r   r   r	   r&      s   z'process_truthfulqa.<locals>._subprocessr'   r)   r   r   r	   process_truthfulqa   s   
rh   c                 C   r   )Nc                 S   s
  d}t ddD ]b}| d|  dkr| d|  }n| d|  dkr+| d|  }ntd	| d
|  d|}| d
|  | d|  | d|  | d|  | d|  | | d}q| d }t| d | d< | | | d< | d | S )Nr   r   r+   winogrande_answer_shot_1winogrande_option1_shot_2winogrande_option2_shot_zAnswer not recognised.winogrande_prompt_shot_rZ   winogrande_idx_shot_r   sentencer   r0   )r   
ValueErrorr4   r   r
   )r    r!   r"   r%   r   rp   r   r   r	   r&      s&   
z'process_winogrande.<locals>._subprocessr'   r)   r   r   r	   process_winogrande   s   
rr   c                 C   s   ddd}|| d  S )Nr   r   )rj   rl   r%   r   )r    answer_to_numr   r   r	   winogrande_doc_to_text   s   
rt   c                 C   s&   | d  dd }| d |d   S )Nrp   rZ   r   )r   r3   r    idxr   r   r	   winogrande_doc_to_target   s   rw   c                    s2    d  d d  d g} fdd|D S )Nrp   rZ   option1option2c                    s    g | ]} d  d | qS )rp   Nr   )r=   optru   r   r	   r@      s     z,winogrande_doc_to_choice.<locals>.<listcomp>)r   )r    optionsr   ru   r	   winogrande_doc_to_choice   s   r|   )r   r5   datasetsstrr
   Datasetr*   r1   rT   rg   rh   rr   rt   rw   r|   r   r   r   r	   <module>   s    9