o
    5ti                  	   @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZ z(d dlZd dlmZmZmZ d dlmZmZmZmZ d dlmZ d dl m!Z! W n	 e"yk   Y nw d dl#Z#d dl$m%Z% d d	l&m'Z' d d
l(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/m0Z0m1Z1 e	rd dl2m3Z3 e4e5Z6eee7e7f ee8 ee8 f Z9G dd deZ:	dde;d de<de7de<fddZ=G dd de-Z>dS )    N)cached_property)TYPE_CHECKINGAny	AwaitableCallableDictIterableListLiteral
NamedTupleOptionalTupleUnion)ClientSessionClientTimeoutTCPConnector)
RetryErrorretrystop_after_attemptwait_exponential)tqdm)tqdm_asyncio	find_spec)BytesIO)utils)Instance)
TemplateLM)Collatorchunksconfigure_pad_token)Imagec                   @   s   e Zd ZU eed< dd ZdS )JsonChatStrpromptc                 C   s   | j |S N)r#   encode)selfencoding r(   M/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/models/api_models.pyr%   9   s   zJsonChatStr.encodeN)__name__
__module____qualname__str__annotations__r%   r(   r(   r(   r)   r"   6   s   
 r"   PNGimgszImage.Imagechatfmtreturnc           	      C   s   g }| D ]&}t  }|j||d t| d}dd| ddd}|| qt|d d	 trA||d d	  |d d	< nd
|d d	 d}||g |d d	< |d 	d |S )a  

    Parameters
    ----------
    img : list[PIL.Image.Image]
        The list of images to encode to base64
    chat : dict
    fmt : str, optional
        Any format Pillow understands (e.g. "PNG", "JPEG").
        Defaults to "PNG".

    Returns
    -------
    dict
    )formatzutf-8	image_urlzdata:image/png;base64,auto)urldetail)typer5   contenttextr9   r<   r9   )
r   savebase64	b64encodegetvaluedecodeappend
isinstancelistpop)	r0   r1   r2   imagesimgbufimg_b64img_dicttext_contentr(   r(   r)   create_image_prompt=   s   rM   c                5       s  e Zd ZdZ																		
								didedededee deed  dedededede	eef dedee dededededee deded ee d!ee d"ed#ed$ee
eef  d%ed&df4 fd'd(Zejddddd)d*e	eee  ee ee ef d+ed,ee ded-ed&efd.d/Z	djd*e	eee  ee ee f d&e	eee  ee ee ef fd0d1Zeej		dkd2e	eee f d3eee  d4ee d&eeeef  fd5d6Zeejd2e	eee f d&ee fd7d8Zed&efd9d:Zed&efd;d<Zed&efd=d>Z	dld?ee
eef  d@ed&e	eeee
 f fdAdBZed&ee fdCdDZed&ee fdEdFZ ed&ee fdGdHZ!			dmdIedJedKedLed&e	eee  ee ee f f
dMdNZ"d3eee  d&ee fdOdPZ#dddQd*e	eee  ee ee f d+ed,ee
 d&ee fdRdSZ$dddddTdUe%dVe&j'd*e	eee  ee ee f d+edWe(dXeee  d,ee
 d&e	ee eeeef  df fdYdZZ)d[e*ee+  d&eeee  ee eeeef  f fd\d]Z,ddd^d_e(dWe(d+edXee d&e	eee  eeeeef   f f
d`daZ-d&eeeef  fdbdcZ.	djd_ee/ dded&ee fdedfZ0	djd_ee/ dded&ee fdgdhZ1  Z2S )nTemplateAPITNhuggingfaceF              main,  model
pretrainedbase_url	tokenizertokenizer_backend)tiktokenrO   remoteNonenonetruncatenum_concurrentmax_retriesmax_gen_toks
batch_sizeseed
max_lengthadd_bos_tokencustom_prefix_token_idtokenized_requeststrust_remote_coderevisionuse_fast_tokenizerverify_certificateca_cert_path
auth_token
eos_stringtimeoutheader
max_imagesr3   c               
      s  t    dd dD }|rtd| d|p|| _|| _|| _|| _t|
ts3d|
v r3t	
d nt|
dkr>t	
d	 |
dkrFt|
nd| _|| _t|	| _t|| _t	d
| d |d | _t|dkrot	d t|| _|dv rzd n|| _|| _|| _|| _t|| _|| _|| _|| _|| _t|| _t|| _t	d| j  | jd u rd | _d| _d S | jd u rI| jdkrdd l}|jj| jr| jn| j|||d| _t | j| _d S | jdkrzdd l!}|"| j| _W n ty } ztd|d }~ww d| jvrt	
d| j d d S d S | jdkrGddl#m$} | js.t%d|| j| j| j| j| j| _t	d| j  d S d S dd l}t|t&sWJ d|jj||||d| _d S )Nc                 S   s   g | ]
}t |d u r|qS r$   r   ).0pkgr(   r(   r)   
<listcomp>   s
    z(TemplateAPI.__init__.<locals>.<listcomp>)aiohttpr   tenacityrequestsz9Attempted to use an API model, but the required packages zd are not installed. Please install these via `pip install lm-eval[api]` or `pip install -e ."[api]"`r6   zQAutomatic batch size is not supported for API models. Defaulting to batch size 1.rP   zgBatch size > 1 detected. Ensure your API supports batched requests with varying total sequence lengths.zUsing max length z - 1zZConcurrent requests are disabled. To enable concurrent requests, set `num_concurrent` > 1.)r^   r_   zUsing tokenizer FrO   r   )rj   rk   use_fastr\   zAttempted to use 'openai' LM type, but the package `tiktoken` is not installed. Please install it via `pip install lm-eval[api]` or `pip install -e .[api]`.openaizPassed `base_url=z` but using (OpenAI) Tiktoken tokenizer backend. Pass `tokenizer_backend=huggingface` and provide the HF tokenizer name if your model does not use Tiktoken.r]   )RemoteTokenizerz1base_url is required for remote tokenizer backendzUsing remote tokenizer from ztokenizer must be a string)'super__init__ModuleNotFoundErrorrW   rY   rZ   _headerrD   inteval_loggerwarning_batch_size	_truncate_max_gen_toks_seedinforf   _concurrentr[   rg   rh   ri   rb   rm   rn   ro   _eos_stringrq   rs   transformersAutoTokenizerfrom_pretrainedr    r\   encoding_for_modellm_eval.utilsr|   
ValueErrorr-   ) r&   rW   rX   rY   rZ   r[   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   kwargsmissing_packagesr   r\   er|   	__class__r(   r)   r~   k   s   
%











zTemplateAPI.__init__generate
gen_kwargsre   eosmessagesr   r   r   c                K      t )zVThis method is responsible for creating the json payload that will be sent to the API.NotImplementedError)r&   r   r   r   re   r   r   r(   r(   r)   _create_payload   s   zTemplateAPI._create_payloadc                 C   sl   t |d tr| jdksJ dt|d jS | js4t |d d tr)| |}| jdkr2|d S |S |S )znHelper method to transform the prompt into the expected API input format. messages consist of batched requestsr   rP   z@non-tokenized chat requests are only supported with batch_size=1)	rD   r"   r   jsonloadsr#   ri   r   decode_batch)r&   r   r   r(   r(   r)   create_message	  s   

zTemplateAPI.create_messageoutputstokensctxlenc                 K   r   )zmMethod used to parse the logprobs from the (batched) API response. This method should return a list of tuplesr   )r   r   r   r   r(   r(   r)   parse_logprobs'  s   	zTemplateAPI.parse_logprobsc                 K   r   )zmMethod used to parse the generations from the (batched) API response. This method should return a list of strr   )r   r   r(   r(   r)   parse_generations2  s   zTemplateAPI.parse_generationsc                 C      dS )zAOverride this property to return the API key for the API request. r(   r&   r(   r(   r)   api_key8  s   zTemplateAPI.api_keyc                 C   s   | j p
dd| j iS )zAOverride this property to return the headers for the API request.AuthorizationzBearer )r   r   r   r(   r(   r)   rr   =  s   zTemplateAPI.headerc                 C   r   )a  Must be defined for LM subclasses which implement Chat Templating.
        Should return the name of the tokenizer or chat template used.
        Used only to properly fingerprint caches when requests are being cached with `--cache_requests`, otherwise not used.
        r   r(   r   r(   r(   r)   tokenizer_nameB  s   zTemplateAPI.tokenizer_namechat_historyadd_generation_promptc                 C   sV   | j dkr| jr| jj|d|| dS | j dkr| jr|S ttjdd |D ddS )zIApplies a chat template to a list of chat history between user and model.rO   F)tokenizer   continue_final_messager]   c                 S   s   g | ]
}i |d diqS r=   r(   )rt   itemr(   r(   r)   rv   [  s    z3TemplateAPI.apply_chat_template.<locals>.<listcomp>)ensure_ascii)r[   ri   rZ   apply_chat_templater"   r   dumps)r&   r   r   r(   r(   r)   r   J  s   zTemplateAPI.apply_chat_templatec                 C   sH   | j d u rd S | jdkr| j jS | jdkr| j jS | jdkr"| j jS d S NrO   r\   r]   )rZ   r[   eos_token_id	eot_tokenr   r(   r(   r)   eot_token_id`  s   



zTemplateAPI.eot_token_idc                 C   sh   | j r| j S | jd ur-| jdkr| jjS | jdkr"| j| jjgS | jdkr+| jjS d S td d S )NrO   r\   r]   ziCannot determine EOS string to pass to stop sequence. Manually set by passing `eos_string` to model_args.)r   rZ   r[   	eos_tokenrB   r   r   r   r   r(   r(   r)   rp   l  s   



zTemplateAPI.eos_stringc                 C   sf   | j d u rd S | jd ur| jS | jdkr"| j jd ur| j jS | j jS | jdkr/| j jp.| j jS | j jS )NrO   r]   )rZ   rh   r[   bos_token_idr   r   r   r(   r(   r)   prefix_token_id}  s   



zTemplateAPI.prefix_token_idstringleft_truncate_lenadd_special_tokens
truncationc                    s  j d u r|gS j dkr7|sj}j|||ddj} r5t|ts. fdd|D }|S |  d  }|S j dkrlt|trHj|}n	fdd|D } rjt|tra|  d  }|S  fdd|D }|S z	j|}W |S  ty   j|}Y |S w )	NrO   F)r   r   return_attention_maskc                       g | ]	}|  d  qS r$   r(   rt   encr   r(   r)   rv         z*TemplateAPI.tok_encode.<locals>.<listcomp>r]   c                    s   g | ]} j |qS r(   )rZ   r%   )rt   sr   r(   r)   rv         c                    r   r$   r(   r   r   r(   r)   rv     r   )	r[   rg   rZ   	input_idsrD   r-   r%   	Exceptionencode_batch)r&   r   r   r   r   r   r'   r(   )r   r&   r)   
tok_encode  sH   





zTemplateAPI.tok_encodec                 C   sF   | j dkr| j|S | j dkr| j|S | j dkr!| j|S d S r   )r[   rZ   batch_decoder   )r&   r   r(   r(   r)   r     s   


zTemplateAPI.decode_batch)r   r   c             	   K   s   t |}z3tj| j| j| |f||| j| jd|| j	| j
d}|js0td|j d |  | W S  tyG   td Y d S w )Nr   )r   headersverifyz'API request failed with error message: . Retrying...zGAPI request failed after multiple retries. Please check the API status.)copydeepcopyry   postrY   r   r   r   rp   rr   rm   okr   r   r<   raise_for_statusr   r   error)r&   r   r   r   r   responser(   r(   r)   
model_call  s:   
	
zTemplateAPI.model_call)r   
cache_keysctxlensr   sessionsemr   r   c             
      s~  t |}| j| |f||| jd|}	|rdnd}
| I d H }zzu|j| j|	| jd4 I d H -}|j	sL|
 I d H }td|j d| d |  | I d H }W d   I d H  n1 I d H sgw   Y  |rt| j|dn| j|||d	}|rt||D ]\}}| j|
|| q|W W |r|  S S  ty } ztd
t| d| d |d }~ww |r|  w w )N)r   r   re   generate_untilloglikelihood)r   r   z!API request failed! Status code: z, Response text: r   )r   r   r   r   z
Exception:z, z, retrying.)r   r   r   r   r   acquirer   rY   rr   r   r<   r   r   statusr   r   r   r   zip
cache_hookadd_partialreleaseBaseExceptionr   repr)r&   r   r   r   r   r   r   r   r   payloadcache_methodacquiredr   
error_textr   answersrescacher   r(   r(   r)   amodel_call  sn   

(

zTemplateAPI.amodel_callr   c                 C   s   g }g }g }|D ]S}|D ]N\}}}|| | j  d  }	t|	t|| k r9tdt| dt| d| j  d t|tdt|t| | j   }
||	 ||
 || qq|||fS )NzContext length (z) + continuation length (z) > max_length (z). Left truncating context.r   )rf   lenr   r   maxrC   )r&   r   inputsr   r   chunk	cache_keycontext_enccontinuation_encinpr   r(   r(   r)   batch_loglikelihood_requests'  s$    


z(TemplateAPI.batch_loglikelihood_requestsr   r   ry   c             	      s   |r|nd gt | }t| j| jd}t| jt|t| jdd4 I d H Kt	t
| jtdddddd	d
 d| j fddtt|| jdt|| jdt|| jdD }tj|ddiI d H W  d   I d H  S 1 I d H syw   Y  d S )N)limitssl)total)	connectorrq         ?rP   
   
multiplierminr   Tc                 S   s   t d| j S )NzRetry attempt )r   r   attempt_number)retry_stater(   r(   r)   <lambda>Q  s    
z2TemplateAPI.get_batched_requests.<locals>.<lambda>)stopwaitreraisebefore_sleepc                    s4   g | ]\}}}t d|| |d qS ))r   r   r   r   r   r   r(   )asynciocreate_task)rt   messager   r   r   r   retry_r   r   r(   r)   rv   V  s    z4TemplateAPI.get_batched_requests.<locals>.<listcomp>ndescRequesting API)r   r   r   rm   r
  	Semaphorer   r   rq   r   r   rb   r   r   r   r   r   r   gather)r&   ry   r   r   r   r   conntasksr(   r  r)   get_batched_requests>  s2   		0z TemplateAPI.get_batched_requestsc              
   K   sP  | j d us	J dg }dtfdd}t||d d}|j| jdkr#| jndd}| jdkrtd	t|d
}|D ]S}| |g\}	}
}t	t
| jtdddddd| j|	dd}t|tr_|g}t| j||	|
d|D ]\}}|d ur|| |d ur| jd|| |d qjq6n| |\}	}
}tjt| j|	|d|
d}||S )NzITokenizer is required for loglikelihood tasks to compute context lengths.reqc                 S   s"   | d | d  }t | t|fS )z%Defines the key for the sorted methodrP      )r   tuple)r  toksr(   r(   r)   _collateq  s   	z3TemplateAPI._loglikelihood_tokens.<locals>._collatesort_fngroup_byrP   r   r  r  r  r   r   r   r   Tr  r  r  F)r   r   r   r   r   )rZ   LogLikelihoodInputsr   get_batchedr   r   r   r   r   r   r   rb   r   r   rD   dictr   r   rC   r   r   update	itertoolschainfrom_iterabler
  runr  get_original)r&   ry   r   r   r  re_ordchunkedpbarr   r   r   r   r   answer_r   r(   r(   r)   _loglikelihood_tokensk  sf   




z!TemplateAPI._loglikelihood_tokensdisable_tqdmc                    sv  g }dd }t |d jdkr=jd u sJ dtdj d tdd	 |D  \} }tfd
d	t||D }ntdd	 |D  \} jrTj	|j
d}nd gt | }dd t| |D }t||dd}|jjdkrwjndd d}jstd jdkr4tdt |d}	|D ]}
t|
 \} }jrЈ d djj fdd|D }tfdd	|D rtdj d d jr|n|}ttjtdddddd j|dt d d!}tj||d"|D ]4\}}|d u rtd# |d$ n|| |d ur,|d ur,jd%| d f| |	 d qqn|D ]}
t|
 \} }jrt d djj fd&d|D }tfd'd	|D rttdj d( d jrz|n|}t!j"#t$%j&| fd)d|D dt d d*}|D ]}|d u rtd+ |d$ q|| qq6|'|S ),Nc                 S   s   t | d  S )Nr   )r   )	_requestsr(   r(   r)   _collate_gen  s   z0TemplateAPI.generate_until.<locals>._collate_genr   r  z7tokenizer is not supported for multimodal requests yet!zUsing max_images z. Set in the model args.c                 s       | ]}|j V  qd S r$   argsrt   r  r(   r(   r)   	<genexpr>      z-TemplateAPI.generate_until.<locals>.<genexpr>c              	   3   s>    | ]\}}t tt|d  d j t|jV  qdS )visualN)r"   r   r   rM   rs   r   r#   )rt   xyr   r(   r)   r7    s    
c                 s   r3  r$   r4  r6  r(   r(   r)   r7    r8  )r   c                 S   s   g | ]
\}}}|||fqS r(   r(   )rt   abcr(   r(   r)   rv     s    z.TemplateAPI.generate_until.<locals>.<listcomp>r   r  rP   )r  batch_fnzLTokenized requests are disabled. Context + generation length is not checked.r  r   rc   c                    r   r$   r(   rt   r:  max_context_lenr(   r)   rv     r   c                 3   "    | ]}t |  jkV  qd S r$   r   rf   r@  rc   r&   r(   r)   r7        
z%Some contexts exceeded (max length: (z) - max_gen_toks: (z). They were left truncated.r   r   r   Tr!  )r   r   r   )r   contextszPAPI returned null content. Check reasoning_content field or generation limits...r   r   c                    r   r$   r(   r@  rA  r(   r)   rv     r   c                 3   rC  r$   rD  r@  rE  r(   r)   r7  !  rF  z) - max_gen_toks (c                    s   g | ]}| d  fqS r   r(   )rt   ctx)all_gen_kwargsr(   r)   rv   -  r   )r   r   r   zNAPI returned null content. Check reasoning_content field or generation limits.)(r   r5  rZ   r   r   rs   r   r  ri   r   rg   r   r#  r   r   r   getr   rf   anyr   r   r   rb   r   r   r   r   r   rC   r   r   r%  r&  r'  r(  r
  r)  r  r*  )r&   ry   r0  r   r2  auxiliary_argsencodings_listr+  r,  r-  r   rG  r  r   generated_textcontextresultsrr(   )rJ  rB  rc   r&   r)   r     s   



	
	

5



	zTemplateAPI.generate_untilc              
   C   s   g }t dd |D |dD ]@\}tttjtj| || j| jd dd}dd |D }| j	|dd}d	d |D }t
|}|| | jd
|f| q|S )Nc                 S   s   g | ]}|j qS r(   r4  r6  r(   r(   r)   rv   D  s    z5TemplateAPI.loglikelihood_rolling.<locals>.<listcomp>)disablerP   )
token_listprefix_tokenmax_seq_lencontext_lenc                 S   s   g | ]}d | qS )r$   r(   r@  r(   r(   r)   rv   S      T)r0  c                 S   s   g | ]}|d  qS rH  r(   r@  r(   r(   r)   rv   [  rX  loglikelihood_rolling)r   rE   mapr   make_disjoint_windowget_rolling_token_windowsr   r   rf   r/  sumrC   r   r   )r&   ry   r0  loglikelihoodsr   rolling_token_windows
string_nllr(   r(   r)   rY  ?  s.   
z!TemplateAPI.loglikelihood_rolling)NNNNrO   FrP   rQ   rR   rP   rS   rT   FNTFrU   TTNNNrV   NrP   )F)NN)T)NFF)3r*   r+   r,   
MULTIMODALr-   r   r
   boolr   r   r   r~   abcabstractmethodr	   r$  r   r"   r   staticmethodr   r   floatr   r   r   r   rr   propertyr   r   r   rp   r   r   r   r   r   r
  r  rE   r   r   r"  r   r  r/  r   r   rY  __classcell__r(   r(   r   r)   rN   h   s   	
 !"$ 	

	&

1
*
	
=
$
"
-@
 rN   )r/   )?rc  r
  r   r&  r   logging	functoolsr   typingr   r   r   r   r   r   r	   r
   r   r   r   r   ry   rw   r   r   r   rx   r   r   r   r   r   tqdm.asyncior   r   r?   importlib.utilr   ior   lm_evalr   lm_eval.api.instancer   lm_eval.api.modelr   lm_eval.models.utilsr   r   r    PILr!   	getLoggerr*   r   r-   r   r"  r"   rE   r$  rM   rN   r(   r(   r(   r)   <module>   sP    8

+