o
    7ti,                     @   s   d dl Z d dlZd dlZd dlZed dZdZdZdd Z	dd	 Z
d
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZdS )    N*   zH{}

{}

Answer with the option's letter from the given choices directly.z6{}

Answer the question using a single word or phrase.Ac                    s4   t  }dd td|D } fdd|D }|S )Nc                 S   s(   g | ]}| d d dd ddqS ) _< >)replace.0img r   L/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/mmmu/utils.py
<listcomp>   s    z doc_to_image.<locals>.<listcomp>z<image [1-7]>c                    s   g | ]} | qS r   r   r
   docr   r   r   %   s    )_doc_to_textrefindall)r   
input_textimage_placeholdersvisualsr   r   r   doc_to_image   s   
r   c                 C   s0   t | }tddD ]}|d| dd}q	|S )z$Get the prompt for a given document.      z<image r   z<image>)r   ranger	   )r   promptir   r   r   doc_to_text*   s   r   c                 C   sz   | d dkr4d}t t| d D ]\}}|dttt|  d| 7 }q| }t| d |}|S t	| d }|S )z\Helper--get the prompt for a given document but DO NOT yet replace <image {i}> with <image>.question_typemultiple-choicer   optionsz
(z) question)
	enumerateastliteral_evalchrord	START_CHRlstripMULTI_CHOICE_EXAMPLE_FORMATformatSHORT_ANS_EXAMPLE_FORMAT)r   choices_strr   choicer   r   r   r   r   6   s   "r   c                 C   s   | d dkr3t | d }g d}|d t| }dd t||D }t|d ||}t| d |}nt|d }t| d |}d	t|iS )
Nr   r    r!   )	r   BCDEFGHIc                 S   s   i | ]\}}||qS r   r   )r   indexansr   r   r   
<dictcomp>S   s    z#process_results.<locals>.<dictcomp>r   answeracc)	r$   r%   lenzipparse_multi_choice_responseeval_multi_choiceparse_open_response	eval_openfloat)r   resultsoption_strsoption_lettersall_choices	index2anspred
is_correctr   r   r   process_resultsL   s   rJ   c                 C   s  dD ]}|  |} qd|  d } d}d}g }|D ]}d| d| v r)|| d}qt|dkrB|D ]}d| d| v rA|| q2t|dkrht|  dkrh| D ]\}}	|	 |  v rg|| d}qTt|dkrut|}
|
S t|d	krg }|r|r|D ]}| d| d}|| qn)|D ]}| d| d}|| qn|D ]}|  ||  }|| q|t	
| }
|
S |d }
|
S )
zl
    Parse the prediction from the generated response.
    Return the predicted index e.g., A, B, C, D.
    ),.!?;:'r   TF()r      r   )stripappendr<   splititemslowerrandomr.   rfindnpargmax)responserF   rG   char	index_ansans_with_brack
candidatesr.   r7   r8   
pred_indexstart_indexescanr   r   r   r>   i   sX   



r>   c                 C   s,   zt | dd W dS  ty   Y dS w )z-
    Check if the given string a number.
    rK   r   TF)rB   r	   
ValueError)stringr   r   r   check_is_number   s   rh   c                 C   sb   |   } t| }|r| dd} t| } t| d} | gS |  } t| dkr.d|  | d gS | gS )zR
    Normalize the str to lower case and make them float numbers if possible.
    rK   r      r   r   )rU   rh   r	   rB   roundrY   r<   )rg   	is_numberr   r   r   normalize_str   s   
rl   c                 C   s@   d}d}d}t || }t || }t || }|| | }|S )z>
    Exact all forms of numbers from a string with regex.
    z-?\b\d{1,3}(?:,\d{3})+\bz-?\d+(?:\.\d+)?[eE][+-]?\d+z3-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d]))r   r   )rg   pattern_commaspattern_scientificpattern_simplenumbers_with_commasnumbers_scientificnumbers_simpleall_numbersr   r   r   extract_numbers   s   rt   c                 C   sl   dd }|| }|  }|D ]	}|t| qg }tt|D ]}|t||  q |}tt|}|S )zn
    Parse the prediction from the generated response.
    Return a list of predicted strings or numbers.
    c                 S   s   g }|    d } td| }g d}g }t|D ]N\}}|t|d kr-|dg d }|D ]*}||v r[|sC||d   }q1t||d   t|k r[||d   }q1|ri|  dvri|| qt|dkrs| gS |S )	NrL   z\.\s(?=[A-Z])|\n)z	could be zso zis zthus z
therefore zfinal zanswer zresult r   =)rP   rK   rL   rM   rN   rO   rP   rQ   r   )rU   rY   r   rW   r#   r<   extendrV   )r^   key_responsessub_responsesindicators_of_keysr7   respshortest_key_response	indicatorr   r   r   get_key_subresponses   s2   


z1parse_open_response.<locals>.get_key_subresponses)copyrw   rt   r   r<   rl   listset)r^   r~   rx   	pred_listr{   tmp_pred_listr   r   r   r   r@      s   1r@   c                 C   s>   d}t | tr| D ]}||krd} |S q	|S | |krd}|S )z.
    Evaluate a multiple choice instance.
    FT)
isinstancer   )gold_ipred_icorrectr:   r   r   r   r?   +  s   
r?   c                 C   s   d}t | trg }| D ]	}|t| qnt| }|D ]'}t |tr8|D ]}t |tr6||v r6|s4d} nq%q||v rC|s@d} |S q|S )z,
    Evaluate an open question instance
    FT)r   r   rw   rl   str)r   r   r   norm_answersr:   rH   norm_ansr   r   r   rA   <  s,   

rA   )r$   rZ   r   numpyr\   seedr*   r,   r(   r   r   r   rJ   r>   rh   rl   rt   r@   r?   rA   r   r   r   r   <module>   s&    
:L