o
    7ti                     @   s   d dl Z d dlZd dlmZ d dlZd dlmZmZ d dlm	Z	m
Z
 dZeeZdedeeejf fdd	Zd
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZdS )    N)	Generator)DEFAULT_SEQ_LENGTHSget_tokenizer)generate_samplesget_haystackzSome special magic {type_needle_v} are hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.
{context}
What are all the special magic {type_needle_v} for {query} mentioned in the provided text?dfreturnc                 C   s$   dt jjttj| t jjdiS )Ntest)split)	datasetsDataset	from_listlist	itertoolschainfrom_iterableSplitTEST)r    r   R/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/ruler/niah_utils.pydownload_dataset   s   r   c                     "     dt}t fdd|D S )Nmax_seq_lengthsc                 3   8    | ]}t td d|td dddtdi  dV  qdS )repeattype_haystackwordsnumbers  max_seq_lengthtemplater   type_needle_ktype_needle_vnum_samples	TOKENIZERNr   r   r   TEMPLATEr   .0seqkwargsr   r   	<genexpr>       
z niah_single_1.<locals>.<genexpr>popr   r   r-   seq_lengthsr   r,   r   niah_single_1      r4   c                     r   )Nr   c                 3   r   )essayr   r   r   r   r    Nr   r'   r)   r,   r   r   r.   *   r/   z niah_single_2.<locals>.<genexpr>r0   r2   r   r,   r   niah_single_2(   r5   r7   c                     r   )Nr   c                 3   r   )r6   r   r   uuidsr   r    Nr   r'   r)   r,   r   r   r.   ;   r/   z niah_single_3.<locals>.<genexpr>r0   r2   r   r,   r   niah_single_39   r5   r9   c                     r   )Nr   c                 3   :    | ]}t td d|td ddddtdi  d	V  qdS )	r6   r   r   r      r   )r!   r"   r   r#   r$   num_needle_kr%   r&   Nr   r'   r)   r,   r   r   r.   L       
z"niah_multikey_1.<locals>.<genexpr>r0   r2   r   r,   r   niah_multikey_1J      r>   c                     r   )Nr   c                 3   r   )needler   r   r   r   r    Nr   r'   r)   r,   r   r   r.   ^   r/   z"niah_multikey_2.<locals>.<genexpr>r0   r2   r   r,   r   niah_multikey_2\   r5   rA   c                     r   )Nr   c                 3   s8    | ]}t td d|td dddtdi  dV  qdS )r@   r   r8   r   r    Nr   r'   r)   r,   r   r   r.   o   r/   z"niah_multikey_3.<locals>.<genexpr>r0   r2   r   r,   r   niah_multikey_3m   r5   rB   c                     r   )Nr   c                 3   r:   )	r6   r   r   r   r;   r   )r!   r"   r   r#   r$   num_needle_vr%   r&   Nr   r'   r)   r,   r   r   r.      r=   z"niah_multivalue.<locals>.<genexpr>r0   r2   r   r,   r   niah_multivalue~   r?   rD   c                     r   )Nr   c                 3   r:   )	r6   r   r   r   r;   r   )r!   r"   r   r#   r$   num_needle_qr%   r&   Nr   r'   r)   r,   r   r   r.      r=   z"niah_multiquery.<locals>.<genexpr>r0   r2   r   r,   r   niah_multiquery   r?   rF   )r   loggingtypingr   r    lm_eval.tasks.ruler.common_utilsr   r    lm_eval.tasks.ruler.prepare_niahr   r   r(   	getLogger__name__eval_loggerdictstrr   r   r4   r7   r9   r>   rA   rB   rD   rF   r   r   r   r   <module>   s"    
