o
    7ti+                     @   s  d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZm	Z	m
Z
 d dlZd dlZd dlmZ d dlmZ d dlmZ zd dlZd dlZd dlmZ W n eyW   edw d	Zd
Zd
ZdZdZe Zejd Z ejd Z!ejd Z"dd e!D Z#e$e%e&e#Z'e%e(ej)d dddd*e+Z,dZ-e j./ddZ0eddde1dee1 fddZ2dd  Z3e3  dFde1fd"d#Z4de1fd$d%Z5de1fd&d'Z6d(e1de1fd)d*Z7d+ed,d-e+d.e	e%e1 e1f d/e1d0e+d1e1d2e+d3e1d4e1d5e+d6e+de8e1e%e1 e1f fd7d8Z9	dGd	d9d+d+d+d	d:dd;d<e+d/e1d1e1d3e1d4e1d=e+d>e+d2e+d0e+d?e+d@e:d6e+de%e; fdAdBZ<ed/e
dC de	e%e1 e1f fdDdEZ=dS )H    N)	lru_cachecache)ListUnionLiteral)parse)version)tqdmsent_tokenizezPlease install the `wonderwords` and `nltk` packages to run this script. You can install them with `pip install lm_eval["ruler"]` or`pip install wonderwords nltk`.   *   ?One of the special magic {type_needle_v} for {key} is: {value}.nouns
adjectivesverbsc                 C   s$   g | ]}t D ]	}| d | qqS )-)r   ).0adjnoun r   T/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/ruler/prepare_niah.py
<listcomp>6   s   $ r   d   (   T)numendpointz3.9.1
LOCAL_RANK0i   )maxsizetextreturnc                 C   s   t | S Nr
   )r!   r   r   r   cached_sent_tokenize@   s   r$   c                  C   sv   t td } t tksJ d|  dt dz	tjd W d	S  ty:   tdkr7td t	d Y d	S Y d	S w )
z)Download 'punkt' if not already installednltkz`nltk` version z is not >= zq. Please update `nltk` before proceeding--older versions are vulnerable to a remote code execution vulnerability.ztokenizers/punkt_tabr   	punkt_tabzDownloaded punkt_tab on rank 0N)
parse_versionr   NLTK_MIN_VERSIONr%   datafindLookupErrorRANKdownloadprint)nltk_versionr   r   r   download_nltk_resourcesE   s   
r0      c                 C   s(   d| d  }d|  d }t t||S )N
      )strrandomrandint)
num_digitslower_boundupper_boundr   r   r   generate_random_numberX   s   r:   c                  C   s   t t} | S r#   )r5   choiceWORDS)wordr   r   r   generate_random_word^   s   
r>   c                   C   s   t tjtdddS )N      )intr   )r4   uuidUUIDr5   getrandbitsr   r   r   r   generate_random_uuidc   s   rE   type_needlec                 C   s8   | dkrt  S | dkrt S | dkrt S t|  d)Nnumberswordsuuids is not implemented.)r:   r>   rE   NotImplementedError)rF   r   r   r   generate_randomg   s   rL   r3   )num_needle_qrandom_seednum_haystackhaystacktype_haystacknum_needle_ktype_needle_knum_needle_vtype_needle_vtemplaterM   rN   c             	      s  d}
g g g }t |D ]-}t g }t |D ]}|t ||
jd |d d q| qt|	| |dkrttsOJ d	d |  }t
|  dgt fddttt|D  t g }g }t d	t|D ])}||d	  }|| }|d	 ||  |d	 t|k r|||d	   qd	|}n>|d
krg|  }n|dkrˇfddt | D }ttt | t|dd}t||D ]
\}}||| qd	|}tt ||}fdd|D }fdd|D }t|d	krd	|d d d |d  n|d }|}|| d	krK|dd}|dd}|dd}|dd}d d |j||d}|||fS )Nr   rU   keyvalueessay r   c                    s    g | ]}t t |d   qS )r   )rA   len)r   depth)document_sentsr   r   r      s    z)generate_input_output.<locals>.<listcomp>r3   repeatneedlec                    s$   g | ]} j ttd qS )rX   )formatrL   )r   _)rP   rS   rU   r   r   r      s    T)reverse
c                    s   g | ]} | qS r   r   )r   i)keysr   r   r      s    c                    s   g | ]} | D ]}|qqS r   r   )r   rf   a)valuesr   r   r      s    z, z, and SomeAzare allisareanswersanswer)rU   contextquery)rangeappendrL   rb   r5   Randomshuffle
isinstancelistjoinr$   stripsortedsampleDEPTHSr]   zipinsertreplace)rO   rP   rQ   rR   rS   rT   rU   rV   rM   rN   NEEDLEneedlesrc   rZ   r!   insertion_positionsdocument_sents_listrf   last_posnext_posrp   	sentencesindexesindexelementindicesqueriesrn   rq   
input_textr   )r_   rP   rg   rS   rU   ri   r   generate_input_outputr   s   

	

r   r?   F)num_samplestokens_to_generaterT   rR   rM   incrementalremove_newline_tabrN   max_seq_lengthr   r   r   r   c                C   s&  |d usJ dt |
|}
g }|}|dkrd}n|dkrd}n|dkr%d}|dkr/|dk r/d}|}d	}|| |k rzt|| ||
||	||||d

\}}}t||d| j}|| |kra||8 }n|dkrp|t| krpt| }n
||7 }|| |k s9tt|d| d| dD ]}|}	 z't|| ||
||	||||d

\}}}t||j| }||ksJ | dW n   ||kr||8 }Y q|rd|dddd 	 }|||||||	 dkrd|d d  d| dnd| d| dd}|d d	 |d vrJ d| d|
| q|S )NzTOKENIZER is not defined.r[   r   r`      ra   i      r   )rQ   rR   rS   rT   rU   rV   rM   rN   r\   zGenerating synthetic samples: z | )descTz exceeds max_seq_length.re   	r3   zThe special magic rW   z for z" mentioned in the provided text isz# mentioned in the provided text are)r   inputoutputslength
max_length
gen_prefixr   r   FzNeedle not in input: z. Something went wrong.)maxr   r]   rx   	input_idsr	   rr   r   ry   splitrs   )rP   	TOKENIZERr   rQ   rS   rU   rV   r   r   rT   rR   rM   r   r   rN   write_jsonsrO   total_tokensr   ro   rq   r   used_haystackr   formatted_outputr   r   r   generate_samples   s   



r   )r[   r`   ra   c                 C   sn   d}| dkr t jdddd }d|}tdd|d}|S | d	kr(d
}|S | dkr0|}|S t|  d)Nr   r[   zbaber/paul_graham_essaystrain)r   r!   r\   z\s+r`   zYThe grass is green. The sky is blue. The sun is yellow. Here we go. There and back again.ra   rJ   )datasetsload_datasetrx   resubr   rK   )rQ   r   r[   rP   r   r   r   get_haystackI  s   
r   )r1   r#   )>osr5   r   rB   	functoolsr   r   typingr   r   r   r   numpynppackaging.versionr   r'   importlib.metadatar   r	   wonderwordsr%   r   ImportErrorNUM_SAMPLESREMOVE_NEWLINE_TAB
STOP_WORDSRANDOM_SEEDr   
RandomWordr_categoriesr   adjsr   rH   rz   rw   setr<   roundlinspaceastyperA   r|   r(   environgetr,   r4   r$   r0   r:   r>   rE   rL   tupler   booldictr   r   r   r   r   r   <module>   s   


"	

g		

r