o
    7ti                     @   sv  d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dlmZm	Z	 ddddZ
d	Ze
d
 ZdZedefddZe	d-deee ee f fddZe	d.deee ee f fddZdededee dee deeee f f
ddZ			 		d/dee dee deded ed!ed"edee fd#d$Zd0dee fd%d&Zdeeejf fd'd(Zd)d* Zd+d, ZdS )1    N)cache)tqdm)DEFAULT_SEQ_LENGTHSget_tokenizer    a  Answer the question based on the given documents. Only give me the answer and do not output any other words.

The following are given documents.

{context}

Answer the question based on the given documents. Only give me the answer and do not output any other words.

Question: {query}Answer:)tokens_to_generatetemplateanswer_prefix*   r	   zDocument {i}:
{document}returnc                 C   s   t | }|  | }|S N)requestsgetraise_for_statusjson)urlresponsedata r   P/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/ruler/qa_utils.pydownload_json$   s   
r   @https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.jsonc                    s   t | }dd |d D }ttt|}dd t|D g }|d D ]?}fdd|d D }|d D ]-  d D ]&}|d	 sa||d
 dd |d D  d  g fdd|D d q;q5q$||fS )Nc                 S   s"   g | ]}|d  D ]}|d qqS )
paragraphscontextr   ).0dpr   r   r   
<listcomp>1      " zread_squad.<locals>.<listcomp>r   c                 S      i | ]\}}||qS r   r   r   idxcr   r   r   
<dictcomp>3       zread_squad.<locals>.<dictcomp>c                    s   g | ]} |d   qS r   r   )r   r   total_docs_dictr   r   r   7   s    r   qasis_impossiblequestionc                 S   s   g | ]}|d  qS )textr   )r   ar   r   r   r   >       answersr   c                    s    g | ]}| d   kr|qS r&   r   r   r"   r   r(   r   r   r   @   s
    )queryoutputsr   more_contextr   sortedlistset	enumerateappend)r   r   
total_docs	total_qasr   	more_docsr)   r   r1   r   
read_squad,   s.   r>   Fhttp://curtis.ml.cmu.edu/datasets/hotpot/hotpot_dev_distractor_v1.jsonc                    s|   t | }dd |D }ttt|}dd t|D  g }|D ]}||d |d g fdd|d D d	 q ||fS )
Nc                 S   s2   g | ]}|d  D ]\}}| dd | qqS )r   
 join)r   r   tr   r   r   r   r   P   s   2 z!read_hotpotqa.<locals>.<listcomp>c                 S   r    r   r   r!   r   r   r   r$   R   r%   z!read_hotpotqa.<locals>.<dictcomp>r+   answerc                    s(   g | ]\}} | d d |  qS )r@   rA   rB   )r   rD   r   r'   r   r   r   Z   s    r   )r2   r3   r   r5   )r   r   r;   r<   r   r   r'   r   read_hotpotqaK   s   

rF   indexnum_docsr)   docsc           
   	      s  ||  d }||  d }||  d  ||   dg |tk rb|t  tkrL fddtD }  t|td|t  t  }n t|t   }fdd|D }n}tt| d		d
d t|D }t
j||d}	|	|fS )Nr2   r3   r   r4   c                    s    g | ]\}}|  vr|qS r   r   r   ir   )	curr_docs	curr_morer   r   r   l   s    z)generate_input_output.<locals>.<listcomp>r   c                    s   g | ]} | qS r   r   r0   )rI   r   r   r   y   r.   z

c                 S   s"   g | ]\}}t j|d  |dqS )   )rK   document)DOCUMENT_PROMPTformatrJ   r   r   r   r      r   )r   r2   )r   lenr9   randomsamplemaxRandomSEEDshufflerC   TEMPLATErQ   )
rG   rH   r)   rI   curr_qcurr_aaddition_docsall_docsr   
input_textr   )rL   rM   rI   r   generate_input_outputc   s4   r_     
   Fmax_seq_lengthnum_samplesr   pre_samplesincrementalc	                 C   sP  g }	|}|}
d}|| |k rDt d|
||d\}}t| |d|  j}|| |kr/|
|8 }
n|
|7 }
|
t|kr>t|}
n|| |k stt|d| dD ]V}|
}	 z"t || |||d\}}t| |j| }||ksuJ | dW n   ||kr||8 }Y qT|rd|ddd	d  }|||||d
d}|		| qO|	S )Nr   )r)   rI    zGenerating QA Samples | )descTz exceeds max_seq_length.r@   	r   )rG   inputr3   length
max_length
gen_prefix)
r_   rR   	input_idsr   rangerC   replacestripsplitr:   )	tokenizerrI   r)   rb   rc   r   rd   re   remove_newline_tabwrite_jsonsrH   total_tokensr^   rE   rG   	used_docsrj   formatted_outputr   r   r   generate_samples   sZ   

rx   c                 K   s    t | }t|||dd|d}|S )Nr`   r   )rr   rI   r)   rc   r   rb   )r   rx   )
pretrainedrI   r)   rb   kwargsrr   rt   r   r   r   get_dataset   s   r{   c                    st   | d| di | dkrt \ nt \  fdd|dtD }dtjjtt	j
|tjjdiS )	Nrr   ry   squadc                 3   s     | ]}t  |d V  qdS ))ry   rI   r)   rb   N)r{   )r   seqrI   ry   r)   r   r   	<genexpr>   s
    
z!get_qa_dataset.<locals>.<genexpr>max_seq_lengthstest)rq   )r   r>   rF   popr   datasetsDataset	from_listr7   	itertoolschainfrom_iterableSplitTEST)dsrz   dfr   r~   r   get_qa_dataset   s   

r   c                  K      t di | S )Nr|   )r|   r   rz   r   r   r   	get_squad      r   c                  K   r   )Nhotpotqa)r   r   r   r   r   r   get_hotpotqa   r   r   )r   )r?   )r`   r   r   ra   Fr   )r   rS   	functoolsr   r   r   r    lm_eval.tasks.ruler.common_utilsr   r   CONFIGrW   rY   rP   dictr   tupler7   strr>   rF   intr_   rx   r{   r   r   r   r   r   r   r   r   <module>   s~   
(

F