o
    6ti                     @   st   d dl Z d dlmZmZmZ d dlmZ e jG dd dZe jG dd dZ	dd	 Z
d
d Zdd Zdd ZdS )    N)DictOptionalUnion)instructions_registryc                   @   sJ   e Zd ZU eed< ee ed< eed< eeeee	eef  f  ed< dS )InputExamplekeyinstruction_id_listpromptkwargsN)
__name__
__module____qualname__int__annotations__liststrr   r   r    r   r   Z/home/ubuntu/.local/lib/python3.10/site-packages/lm_eval/tasks/leaderboard/ifeval/utils.pyr      s
   
 $r   c                   @   s>   e Zd ZU ee ed< eed< eed< eed< ee ed< dS )OutputExampler   r	   responsefollow_all_instructionsfollow_instruction_listN)r   r   r   r   r   r   boolr   r   r   r   r      s   
 r   c           
      C   s   | j }g }t|D ]F\}}tj| }||}dd | j|  D }|jd	i | | }	|	r;d|	v r;|j| jd |	 rJ|
|rJ|d q	|d q	t| j | j|t||dS )
z3Tests response to see if instructions are followed.c                 S      i | ]	\}}|r||qS r   r   .0kvr   r   r   
<dictcomp>%       z5test_instruction_following_strict.<locals>.<dictcomp>r	   r	   TFr   r	   r   r   r   Nr   )r   	enumerater   INSTRUCTION_DICTr
   itemsbuild_descriptionget_instruction_argsr	   stripcheck_followingappendr   all)
inpr   instruction_listis_following_listindexinstruction_idinstruction_clsinstructionr
   argsr   r   r   !test_instruction_following_strict   s(   
r3   c                 C   sT  | d}d|dd  }d|dd  }d|dd  }|dd}|dd}|dd}|dd}	||||||||	g}
| j}g }t|D ]K\}}tj| }||}dd | j| 	 D }|j
di | | }|rd	|v r|j
| jd
 d}|
D ]}| r||rd} nq|| qQt| j| j|t||dS )z=Tests response for an upper bound for following instructions.
   N* c                 S   r   r   r   r   r   r   r   r   X   r   z4test_instruction_following_loose.<locals>.<dictcomp>r	   r    FTr!   r   )splitjoinr'   replacer   r"   r   r#   r
   r$   r%   r&   r	   r(   r)   r   r*   )r+   r   rresponse_remove_firstresponse_remove_lastresponse_remove_bothrevised_responserevised_response_remove_firstrevised_response_remove_lastrevised_response_remove_bothall_responsesr,   r-   r.   r/   r0   r1   r
   r2   is_followingr   r   r    test_instruction_following_loose9   sR   


rF   c                 C   sR   t | d | d | d | d d}|d }t||}t||}|j|j|j|jdS )Nr   r   r	   r
   )r   r   r	   r
   r   )prompt_level_strict_accinst_level_strict_accprompt_level_loose_accinst_level_loose_acc)r   r3   rF   r   r   )docresultsr+   r   
out_strict	out_looser   r   r   process_resultso   s   

rO   c                 C   s"   dd | D }t |t| }|S )Nc                 S   s   g | ]	}|D ]}|qqS r   r   )r   sublistitemr   r   r   
<listcomp>   r   z&agg_inst_level_acc.<locals>.<listcomp>)sumlen)r$   
flat_itemsinst_level_accr   r   r   agg_inst_level_acc   s   rW   )dataclassestypingr   r   r   lm_eval.tasks.ifevalr   	dataclassr   r   r3   rF   rO   rW   r   r   r   r   <module>   s    !6