o
    7ti#                     @   s   d dl Z d dlZd dlZd dlZdedefddZdejdejfddZdejdejfd	d
ZdejdejfddZ	dejdejfddZ
dejdejfddZdd Zdd Zdd ZdS )    Nstringreturnc                 C   s   t | d S )Nzutf-8)hashlibsha256encode	hexdigest)r    r   `/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/metabench/process_docs_permute.pyhash_string   s   r
   datasetc                 C      dd }|  |S )Nc           
      S   sL  d}t ddD ]R}| d|  }| d|  | d|  }| d|  | d|  d |}| d|  d | }| d|  | d	|  | d
| d| d}q|| d< | d t| d | d< | d d | d }| d d | }|}	|	|u rt| d d  | d d |}	|	|u s| d d |	 | d< | S )N       arc_question_shot_arc_answerKey_shot_arc_choices_shot_labeltextarc_idx_shot_z
Question: 	
Answer: 

twentyfive_shot_prepromptalltwentyfiveshot_longpromptquestionoriginal_hashchoices	answerKey)rangepopindexr
   randomshuffle)
doclong_promptshotr   
answer_lab
answer_idxansweroriginal_answer_idxcorrect_answer_textnew_answer_idxr   r   r	   _subprocess   s.   
z process_arc.<locals>._subprocessmapr   r,   r   r   r	   process_arc   s   
r0   c                    s   dd   fdd}|  |S )Nc                 S   s2   |   } | dd} tdd| } | dd} | S )Nz [title]z. z\[.*?\]r   z   )stripreplaceresub)r   r   r   r	   process_txt-   s
   z&process_hellaswag.<locals>.process_txtc           	         s0  | d d | d    }| d | d | d  | d d | | d< t| d }| d	 | }|}||u rKt| d	  | d	 |}||u s9t|| d<  fd
d| d	 D | d< t| d | d< | d | d	 d}tddD ]}| d|  d | d|     }| d|  | d|  | d|   | d|  d | } | d|  t| d|   }| d|  | d|  | d|  | | d| d}| d|  | d|  | d|  | d|  qwt| d | d< || d< | d | S )Nctx_ar1   ctx_bctxactivity_labelz: queryr   endingsc                    s   g | ]} |qS r   r   ).0endingr6   r   r	   
<listcomp>E   s    z:process_hellaswag.<locals>._preprocess.<locals>.<listcomp>r   goldr   r      hellaswag_ctx_a_shot_hellaswag_ctx_b_shot_hellaswag_ctx_shot_hellaswag_activity_labels_shot_hellaswag_endings_shot_hellaswag_label_shot_r   hellaswag_ind_shot_hellaswag_source_id_shot_hellaswag_split_shot_hellaswag_split_type_shot_r   ten_shot_prepromptalltenshot_longprompt)	
capitalizer   intr!   r"   r    strr   r
   )	r#   r9   r)   r*   r+   r$   r%   r   r>   r?   r   r	   _preprocess5   s`   





z&process_hellaswag.<locals>._preprocessr-   )r   rR   r   r?   r	   process_hellaswag,   s   
5rS   c                 C   r   )Nc                 S   s  g d}dd | d d d}tddD ]p}| d	|   }| d	|  |t| d
|   }| d|  d }| d|  d }| d|  d }| d|  d }	| d|  | d
|  | d|  | | d| d| d| d|	 d| d}qt| d | d< || d< | d t| d }
| d |
 }|
}||
u rt| d  | d 	|}||
u s|| d< | S )N)ABCDzAThe following are multiple choice questions (with answers) about r1   subject_z.

r      mmlu_question_shot_mmlu_answers_shot_mmlu_choices_shot_r         mmlu_ind_shot_z
A. z
B. z
C. z
D. r   r   r   r   five_shot_prepromptallfiveshot_longpromptr(   r   )
joinsplitr   r2   r   rP   r
   r!   r"   r    )r#   r   r$   r%   r   r(   choice_Achoice_Bchoice_Cchoice_Dr)   r*   r+   r   r   r	   r,   n   s4   .
z!process_mmlu.<locals>._subprocessr-   r/   r   r   r	   process_mmlum   s   
"ri   c                 C   r   )Nc                 S   s   d}| d d | }|}||u r&t | d d  | d d |}||u sdgt| d d  }d||< t| d | d< || d d< || d< | S )	Nr   mc1_targetsr   labelsr   r   r   r(   )r!   r"   r    lenr
   )r#   r)   r*   r+   rk   r   r   r	   r,      s   z'process_truthfulqa.<locals>._subprocessr-   r/   r   r   r	   process_truthfulqa   s   
rm   c                 C   r   )Nc                 S   sV  d}t ddD ]b}| d|  dkr| d|  }n| d|  dkr+| d|  }ntd	| d
|  d|}| d
|  | d|  | d|  | d|  | d|  | | d}q| d }t| d | d< | | | d< | d | d }| d }| d }|| d< || d< |dkrd| d< | S |dkrd| d< | S )Nr   r   rZ   winogrande_answer_shot_1winogrande_option1_shot_2winogrande_option2_shot_zAnswer not recognised.winogrande_prompt_shot_rY   winogrande_idx_shot_r   sentencer   rb   option1option2r(   )r   
ValueErrorr3   r   r
   )r#   r$   r%   r(   r   ru   rv   rw   r   r   r	   r,      s:   
z'process_winogrande.<locals>._subprocessr-   r/   r   r   r	   process_winogrande   s   
'ry   c                 C   s   ddd}|| d  S )Nr   r   )ro   rq   r(   r   )r#   answer_to_numr   r   r	   winogrande_doc_to_text   s   
r{   c                 C   s&   | d  dd }| d |d   S )Nru   rY   r   )r    r2   r#   idxr   r   r	   winogrande_doc_to_target   s   r~   c                    s2    d  d d  d g} fdd|D S )Nru   rY   rv   rw   c                    s    g | ]} d  d | qS )ru   Nr   )r=   optr|   r   r	   r@      s     z,winogrande_doc_to_choice.<locals>.<listcomp>)r    )r#   optionsr   r|   r	   winogrande_doc_to_choice   s   r   )r   r!   r4   datasetsrQ   r
   Datasetr0   rS   ri   rm   ry   r{   r~   r   r   r   r   r	   <module>   s     A&+