o 5tiT@sdZddlZddlZddlZddlZddlZddlmZmZm Z m Z m Z m Z ddl ZddlZddlmZmZmZddlmZddlmZddlmZddlmZmZmZdd lm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(d d l)m*Z*d dl+m,Z,d dl-m.Z.d dl/m0Z0e0e1Z2Gddde"Z3ddZ4GdddZ5Gddde5Z6Gddde6Z7Gddde6Z8Gddde6Z9Gdd d Z:d$d"d#Z;dS)%z EvaluationModule base class.N)AnyDictListOptionalTupleUnion) DatasetInfoDownloadConfigDownloadManager)Dataset) ArrowReader) ArrowWriter)FeaturesSequenceValue)#_check_non_null_non_empty_recursive) BaseFileLockFileLockTimeout)copyfunc temp_seedzip_dict)config)EvaluationModuleInfo)camelcase_to_snakecase) get_loggercsBeZdZdZfddZddZddZedefd d Z Z S) FileFreeLockz-Thread lock until a file **cannot** be lockedcs.t||_tj|g|Ri|d|_dSN)rfilelocksuper__init__ _lock_file_fd)self lock_fileargskwargs __class__C/home/ubuntu/.local/lib/python3.10/site-packages/evaluate/module.pyr!/s  zFileFreeLock.__init__cCsHz |jjdddWnty|jj|_YdSw|jd|_dS)Ng{Gz?g{Gz?)timeout poll_interval)racquirerr$r"releaser#r)r)r*_acquire4s   zFileFreeLock._acquirecCs d|_dSrr"r/r)r)r*_release? zFileFreeLock._releasereturncCs |jduSrr1r/r)r)r* is_lockedBs zFileFreeLock.is_locked) __name__ __module__ __qualname____doc__r!r0r2propertyboolr5 __classcell__r)r)r'r*r,s  rcCsPt|tus t|dkr|Sdd}d||ddd||dddS) NcSsddd|DS)Nz, css|]}t|VqdSr)repr).0xr)r)r* Nz?summarize_if_long_list..format_chunk..)join)chunkr)r)r* format_chunkMsz,summarize_if_long_list..format_chunk[z, ..., ])typelistlen)objrEr)r)r*summarize_if_long_listIs*rNc@s0eZdZdZdefddZeddZedefdd Z ede efd d Z edefd d Z edefddZ edefddZedefddZede efddZedefddZede eefddZede eefddZedefddZede efddZedefd d!Zd"S)#EvaluationModuleInfoMixinzThis base class exposes some attributes of EvaluationModuleInfo at the base level of the EvaluationModule for easy access. infocCs ||_dSr _module_info)r#rPr)r)r*r!Xr3z"EvaluationModuleInfoMixin.__init__cCs|jS)zc:class:`evaluate.EvaluationModuleInfo` object containing all the metadata in the evaluation module.rQr/r)r)r*rP[szEvaluationModuleInfoMixin.infor4cC|jjSr)rR module_namer/r)r)r*name`zEvaluationModuleInfoMixin.namecCrSr)rR experiment_idr/r)r)r*rWdrVz'EvaluationModuleInfoMixin.experiment_idcCrSr)rR descriptionr/r)r)r*rXhrVz%EvaluationModuleInfoMixin.descriptioncCrSr)rRcitationr/r)r)r*rYlrVz"EvaluationModuleInfoMixin.citationcCrSr)rRfeaturesr/r)r)r*rZprVz"EvaluationModuleInfoMixin.featurescCrSr)rRinputs_descriptionr/r)r)r*r[trVz,EvaluationModuleInfoMixin.inputs_descriptioncCrSr)rRhomepager/r)r)r*r\xrVz"EvaluationModuleInfoMixin.homepagecCrSr)rRlicenser/r)r)r*r]|rVz!EvaluationModuleInfoMixin.licensecCrSr)rR codebase_urlsr/r)r)r*r^rVz'EvaluationModuleInfoMixin.codebase_urlscCrSr)rRreference_urlsr/r)r)r*r_rVz(EvaluationModuleInfoMixin.reference_urlscCrSr)rR streamabler/r)r)r*r`rVz$EvaluationModuleInfoMixin.streamablecCrSr)rRformatr/r)r)r*rarVz EvaluationModuleInfoMixin.formatcCrSr)rR module_typer/r)r)r*rbrVz%EvaluationModuleInfoMixin.module_typeN)r6r7r8r9rr!r:rPstrrUrrWrXrYrrZr[r\r]rr^r_r;r`rarbr)r)r)r*rOSs> rOc@seZdZdZ          dCdeed ed eed ed ed eedeedededeee ffddZ ddZ ddZ ddZ dDdeeeffddZdeeeeeffddZdd Zd!d"Zd#d$Zddd%deefd&d'Zddd%d(d)Zddd*d+d,Zd-d.Zd/d0Zd1d2ZdDd3d4Zdefd5d6Z  dEd7ee d8ee!fd9d:Z"d;d<Z#ddd%de$ee%ffd=d>Z&d?d@Z'dAdBZ(dS)FEvaluationModuleaaA `EvaluationModule` is the base class and common API for metrics, comparisons, and measurements. Args: config_name (`str`): This is used to define a hash specific to a module computation script and prevents the module's data to be overridden when the module loading script is modified. keep_in_memory (`bool`): Keep all predictions and references in memory. Not possible in distributed settings. cache_dir (`str`): Path to a directory in which temporary prediction/references data will be stored. The data directory should be located on a shared file-system in distributed setups. num_process (`int`): Specify the total number of nodes in a distributed settings. This is useful to compute module in distributed setups (in particular non-additive modules like F1). process_id (`int`): Specify the id of the current process in a distributed setup (between 0 and num_process-1) This is useful to compute module in distributed setups (in particular non-additive metrics like F1). seed (`int`, optional): If specified, this will temporarily set numpy's random seed when [`~evaluate.EvaluationModule.compute`] is run. experiment_id (`str`): A specific experiment id. This is used if several distributed evaluations share the same file system. This is useful to compute module in distributed setups (in particular non-additive metrics like F1). hash (`str`): Used to identify the evaluation module according to the hashed file contents. max_concurrent_cache_files (`int`): Max number of concurrent module cache files (default `10000`). timeout (`Union[int, float]`): Timeout in second for distributed setting synchronization. NFrr'd config_namekeep_in_memory cache_dir num_process process_idseedrWhashmax_concurrent_cache_filesr+c Ks|pd|_|} t|jj| _|j| _|pd| _t|| t |t r(|dkr,t dt |t r5||kr9t d|rC|dkrCt d||_ ||_ | |_||_tj|pVtj|_||_|durytj^} }}} |dkrs||n|d|_n||_| |_tt|j ||_ tt|j!||_!tt|j"||_"|j j#j$|j%j&7_$|j!j#j$|j%j&7_$|j"j#j$|j%j&7_$d|_'d|_(d|_)d|_*d|_+d|_,d|_-d|_.d|_/d|_0||_1dS) Ndefaultdefault_experimentrz.'process_id' should be a number greater than 0z8'num_process' should be a number greater than process_idrzPUsing 'keep_in_memory' is not possible in distributed setting (num_process > 1).ip)2rg_inforr(r6rTrWrOr! isinstanceint ValueErrorrjrkrnrhospath expanduserrHF_METRICS_CACHE_data_dir_root_build_data_dirdata_dirnprandom get_staterlr+types MethodTypercompute add_batchadd__func__r9rPr[selected_feature_format buf_writerwriterwriter_batch_sizedatacache_file_namerrendez_vous_lock file_paths filelocks_hash)r#rgrhrirjrkrlrWrmrnr+r&rP_posr)r)r*r!sP      zEvaluationModule.__init__cCs|jdurdSt|jS)zReturn the number of examples (predictions or predictions/references pair) currently stored in the evaluation module's cache. Nr)rrLr/r)r)r*__len__szEvaluationModule.__len__c Cs0d|jd|jd|jd|jdt|d S)NzEvaluationModule(name: "z", module_type: "z ", features: z , usage: """z""", stored examples: ))rUrbrZr[rLr/r)r)r*__repr__szEvaluationModule.__repr__cCs,|j}tj||j|j}tj|dd|S)a Path of this evaluation module in cache_dir: Will be: self._data_dir_root/self.name/self.config_name/self.hash (if not none)/ If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. T)exist_ok)ryrurvrCrUrgmakedirs)r#builder_data_dirr)r)r*rz sz EvaluationModule._build_data_dirr4cCstj|j|jd|jd|jd}d}t|jD]^}t |d}z|j |dWnIt yt|jdkrDt d|d|jd d||jdkrUt d |jd dt t}tj|j|jd|d|jd|jd}Yqw||fS||fS) zTCreate a new cache file. If the default cache file is used, we generated a new hash.-.arrowN.lockr+rziError in _create_cache_file: another evaluation module instance is already using the local cache file at .. Please specify an experiment_id (currently: E) to avoid collision between distributed evaluation module instances.zCannot acquire lock, too many evaluation module instance are operating concurrently on this file system.You should set a larger value of max_concurrent_cache_files when creating the evaluation module (current value is z).)rurvrCr{rWrjrkrangernrr-rrtrcuuiduuid4)r#r+ file_pathri file_uuidr)r)r*_create_cache_files<(    $z#EvaluationModule._create_cache_filec sjdkrjdurtdjg}n fddtjD}g}t|D]7\}}|dkr4|jq%t|d}z |jj dWnt yVtd |d |d dw||q%||fS) zGet a lock on all the cache files in a distributed setup. We wait for timeout second to let all the distributed node finish their tasks (default is 100 seconds). rNzEvaluation module cache file doesn't exist. Please make sure that you call `add` or `add_batch` at least once before calling `compute`.c 2g|]}tjjjdjd|dqS)rrrurvrCr{rWrjr?rkr/r)r* B$z9EvaluationModule._get_all_cache_files..rrrz#Cannot acquire lock on cached file z for process .) rjrrtr enumerateappendrrr-r+r)r#rrrkrrr)r/r*_get_all_cache_files6s2       z%EvaluationModule._get_all_cache_filesc srfddtjD}|D](}t|}z |jjdWnty1td|djddw|qdS)Nc r)rz .arrow.lockrrr/r)r*rZrz?EvaluationModule._check_all_processes_locks..rExpected to find locked file  from process  but it doesn't exist.) rrjrr-r+rrtrkr.)r#expected_lock_file_namesexpected_lock_file_name nofilelockr)r/r*_check_all_processes_locksYs   z+EvaluationModule._check_all_processes_locksc Cstj|j|jd|jd}t|}z |j|jdWnt y1t d|d|j ddw| tj|j|jd|jd}t |}z |j|jdWnt ygt d|d|j d dw| dS) Nrz -0.arrow.lockrrrr -rdv.lockzCouldn't acquire lock on r)rurvrCr{rWrjrr-r+rrtrkr.r)r#rrlock_file_namerr)r)r*_check_rendez_vousis(     z#EvaluationModule._check_rendez_vouscCs|jdur |jd|_|jdur|jdkr|j|jr5t|jt|j dd}t |j |_dS|jdkrq|\}}ztdt|j dd}t d i|dd|D|_Wn tyhtddw||_||_dSdS) zClose all the writing process and load/gather the data from all the nodes if main node or all_process is True. Nr)rZ)rvrPcSsg|]}d|iqS)filenamer))r?fr)r)r*rz.EvaluationModule._finalize..zError in finalize: another evaluation module instance is already using the local cache file. Please specify an experiment_id to avoid collision between distributed evaluation module instances.r))rfinalizerrkr.rhr r{rrr from_bufferrgetvaluerr read_filesFileNotFoundErrorrtrr)r#readerrrr)r)r* _finalize}s.     $  zEvaluationModule._finalize predictions referencesc  s||d|dur |dur fddD}|nfddD}|r;td|dtfd dD}fd dD}td d |Drcjdi|d_d_ d_ j d krj j jjdfddD}tjjdi||}Wdn1swYjdurd_` d_ |StttjjD]\} } td| ` d_ `d_t| | q|SdS)aCompute the evaluation module. Usage of positional arguments is not allowed to prevent mistakes. Args: predictions (`list/array/tensor`, *optional*): Predictions. references (`list/array/tensor`, *optional*): References. **kwargs (optional): Keyword arguments that will be forwarded to the evaluation module [`~evaluate.EvaluationModule.compute`] method (see details in the docstring). Return: `dict` or `None` - Dictionary with the results if this evaluation module is run on the main process (`process_id == 0`). - `None` if the evaluation module is not run on the main process (`process_id != 0`). ```py >>> import evaluate >>> accuracy = evaluate.load("accuracy") >>> accuracy.compute(predictions=[0, 1, 1, 0], references=[0, 1, 0, 1]) ``` rNcsi|] }|vr|dqSrr)r?k all_kwargsr)r* sz,EvaluationModule.compute..csg|]}|vr|qSr)r)rrr)r*rz,EvaluationModule.compute..z&Evaluation module inputs are missing: . All required inputs are ci|]}||qSr)r)r? input_namerr)r*rcs"i|] }|vr||qSr)_feature_namesr)r&r#r)r*rs"css|]}|duVqdSrr))r?vr)r)r*rArBz+EvaluationModule.compute..r)rJcs i|] }|j|ddqSr)rrr/r)r*r z Removing r))rupdatertrKanyvaluesrrrrrrkr set_formatrPrarrl_computerreversedziprrloggerrruremover.) r#rrr&missing_kwargsmissing_inputsinputscompute_kwargsoutputrrr))rr&r#r*rsL       zEvaluationModule.computec sfdd|D}|rtd|dt||d|fddDjdur:_z)D]\}}t|d krT j||d q?j j WdSt j tfytfd d Drttfd dDd }d dtd|dt|d }n2tjddhkrdjd}dfdd jD} || 7}ndjdt|dt|}t|dw)aAdd a batch of predictions and references for the evaluation module's stack. Args: predictions (`list/array/tensor`, *optional*): Predictions. references (`list/array/tensor`, *optional*): References. Example: ```py >>> import evaluate >>> accuracy = evaluate.load("accuracy") >>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): ... accuracy.add_batch(references=refs, predictions=preds) ``` cg|] }|vr|qSr)rrr/r)r*rz.EvaluationModule.add_batch.."Bad inputs for evaluation module: rrcrr)r)rbatchr)r*rrz.EvaluationModule.add_batch..Nrc3s0|]}t|tttkVqdSr)rLnextiterrr?crr)r*rA s.z-EvaluationModule.add_batch..cs(g|]}t|tkr|qSr))rLr)rcol0r)r*r s(zMismatch in the number of z (z) and rrrz@Module inputs don't match the expected format. Expected format: , c3(|]}d|dt|VqdSzInput : NrNrrr)r*rA  zPPredictions and/or references don't match the expected format. Expected format: , Input predictions: , Input references: )rtrKrr_infer_feature_from_batchr _init_writeritemsrL_enforce_nested_string_type encode_batch write_batchpa ArrowInvalid TypeErrorrrrsetrCrN) r#rrr& bad_inputskeycolumnbad_col error_msgerror_msg_inputsr))rrr#r*rsT     ,  zEvaluationModule.add_batch) prediction referencec sfdd|D}|rtd|dt||d|fddDjdur:_zjjj WdSt j t fywd jd }d fd d jD}||7}t|dw) aAdd one prediction and reference for the evaluation module's stack. Args: prediction (`list/array/tensor`, *optional*): Predictions. reference (`list/array/tensor`, *optional*): References. Example: ```py >>> import evaluate >>> accuracy = evaluate.load("accuracy") >>> accuracy.add(references=[0,1], predictions=[1,0]) ``` crr)rrr/r)r*r5rz(EvaluationModule.add..rrrcrr)r)rexampler)r*r;rz(EvaluationModule.add..NzKEvaluation module inputs don't match the expected format. Expected format: rc3rrrrrr)r*rAHrz'EvaluationModule.add..)rtrKrr_infer_feature_from_examplerrrencode_examplewriterrrrC)r#rrr&rrrr))rr#r*r$s2    zEvaluationModule.addcCs2t|jtr |jStdd|D}||S)NcSsg|] \}}||dfqSrr))r?rrr)r)r*rSrz>EvaluationModule._infer_feature_from_batch..)rrrZrdictrr)r#rrr)r)r*rOs  z*EvaluationModule._infer_feature_from_batchc Cst|jtr |jS|jD]}z||||||WSttfy)Yq wdddt|jD}d|dt |ddt |d}t|d) N cSs g|] \}}d|d|qS)zFeature option rr))r?rfeaturer)r)r*rarz@EvaluationModule._infer_feature_from_example..zPPredictions and/or references don't match the expected format. Expected format: rrrr) rrrZrrr rtrrCrrN)r#rrZfeature_stringsrr)r)r*rVs(        z,EvaluationModule._infer_feature_from_examplecCs4t|jtrt|jd}|St|j}|S)Nr)rrrZrKkeys)r# feature_namesr)r)r*rjs zEvaluationModule._feature_namesc Cs$|jdkr=|jdkr=tj|j|jd|jd}t||_z |jj |dWnt y<t d|d|jddw|j rQt |_t|j|j|jd |_n$d|_|jdus^|jdurj|\}}||_||_t|j|j|jd |_|jdkr|jdkr||jdS|dSdS) NrrrrrzbError in _init_writer: another evalution module instance is already using the local cache file at rr)rZstreamr)rZrvr)rjrkrurvrCr{rWrrr- TimeoutErrorrtrhrBufferOutputStreamrr rrrrrrrr.r)r#r+rrrr)r)r*rqsH            zEvaluationModule._init_writercCt)a-Construct the EvaluationModuleInfo object. See `EvaluationModuleInfo` for details. Warning: This function is only called once and the result is cached for all following .info() calls. Returns: info: (EvaluationModuleInfo) The EvaluationModule information NotImplementedErrorr/r)r)r*rq zEvaluationModule._infodownload_config dl_managercCsN|dur |durt}tj|jd|_d|_t|j||jd}| |dS)a|Downloads and prepares evaluation module for reading. Args: download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. dl_manager ([`DownloadManager`], *optional*): Specific download manager to use. Example: ```py >>> import evaluate ``` N downloadsF) dataset_namerr{) r rurvrCr{riforce_downloadr rU_download_and_prepare)r#rrr)r)r*download_and_prepares z%EvaluationModule.download_and_preparecCsdS)aoDownloads and prepares resources for the evaluation module. This is the internal implementation to overwrite called when user calls `download_and_prepare`. It should download all required resources for the evaluation module. Args: dl_manager (:class:`DownloadManager`): `DownloadManager` used to download and cache data. Nr))r#rr)r)r*rrz&EvaluationModule._download_and_preparecKr)zOThis method defines the common API for all the evaluation module in the libraryr)r#rrr&r)r)r*rszEvaluationModule._computecCs`t|dr|jdur|jt|dr|jdur|jt|dr%|`t|dr.|`dSdS)Nrrrr)hasattrrr.rrrr/r)r)r*__del__s    zEvaluationModule.__del__cst|trfddt||DSt|ttfr&|dfdd|DSt|trt|jtrt|ttfrbt|jg|RD]\}}|ddD]}t||dr^|d|nqKqAdSt|j|D]\}\}|D]}t|r|nqpqhdSt|t rt d|d|durdSt |dkr|D] }t||jrnqt|ts|j|SdSdSt|t rt j|jrt|t std t|d dSdSdS) a Recursively checks if there is any Value feature of type string and throws TypeError if corresponding object is not a string. Since any Python object can be cast to string this avoids implicitly casting wrong input types (e.g. lists) to string without error. cs g|] \}\}}||qSr)r)r?r sub_schemaor/r)r*rrz@EvaluationModule._enforce_nested_string_type..rcsg|]}|qSr)r")r?r$r#r#r)r*rrrNz+Got a string but expected a list instead: ''zExpected type str but got r)rrr rrKtuplerrrrrcrtrLrrr is_stringpa_typerrJ)r#schemarMr dict_tuplessub_objsub_objs first_elmtr)r%r*rsT          z,EvaluationModule._enforce_nested_string_type) NFNrrNNNrerfrNN))r6r7r8r9rrcr;rsrfloatr!rrrzrrrrrrrrr rrrrrrrrrqr r rrrrrr!rr)r)r)r*rdsx    I !#"I<+ )    rdc@eZdZdZdS)MetricaA Metric is the base class and common API for all metrics. Args: config_name (`str`): This is used to define a hash specific to a metric computation script and prevents the metric's data to be overridden when the metric loading script is modified. keep_in_memory (`bool`): Keep all predictions and references in memory. Not possible in distributed settings. cache_dir (`str`): Path to a directory in which temporary prediction/references data will be stored. The data directory should be located on a shared file-system in distributed setups. num_process (`int`): Specify the total number of nodes in a distributed settings. This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). process_id (`int`): Specify the id of the current process in a distributed setup (between 0 and num_process-1) This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). seed (`int`, *optional*): If specified, this will temporarily set numpy's random seed when [`~evaluate.Metric.compute`] is run. experiment_id (`str`): A specific experiment id. This is used if several distributed evaluations share the same file system. This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). max_concurrent_cache_files (`int`): Max number of concurrent metric cache files (default `10000`). timeout (`Union[int, float]`): Timeout in second for distributed setting synchronization. Nr6r7r8r9r)r)r)r*r3r3c@r2) ComparisonaA Comparison is the base class and common API for all comparisons. Args: config_name (`str`): This is used to define a hash specific to a comparison computation script and prevents the comparison's data to be overridden when the comparison loading script is modified. keep_in_memory (`bool`): Keep all predictions and references in memory. Not possible in distributed settings. cache_dir (`str`): Path to a directory in which temporary prediction/references data will be stored. The data directory should be located on a shared file-system in distributed setups. num_process (`int`): Specify the total number of nodes in a distributed settings. This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). process_id (`int`): Specify the id of the current process in a distributed setup (between 0 and num_process-1) This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). seed (`int`, *optional*): If specified, this will temporarily set numpy's random seed when [`~evaluate.Comparison.compute`] is run. experiment_id (`str`): A specific experiment id. This is used if several distributed evaluations share the same file system. This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). max_concurrent_cache_files (`int`): Max number of concurrent comparison cache files (default `10000`). timeout (`Union[int, float]`): Timeout in second for distributed setting synchronization. Nr4r)r)r)r*r6,r5r6c@r2) MeasurementaA Measurement is the base class and common API for all measurements. Args: config_name (`str`): This is used to define a hash specific to a measurement computation script and prevents the measurement's data to be overridden when the measurement loading script is modified. keep_in_memory (`bool`): Keep all predictions and references in memory. Not possible in distributed settings. cache_dir (`str`): Path to a directory in which temporary prediction/references data will be stored. The data directory should be located on a shared file-system in distributed setups. num_process (`int`): Specify the total number of nodes in a distributed settings. This is useful to compute measurements in distributed setups (in particular non-additive measurements). process_id (`int`): Specify the id of the current process in a distributed setup (between 0 and num_process-1) This is useful to compute measurements in distributed setups (in particular non-additive measurements). seed (`int`, *optional*): If specified, this will temporarily set numpy's random seed when [`~evaluate.Measurement.compute`] is run. experiment_id (`str`): A specific experiment id. This is used if several distributed evaluations share the same file system. This is useful to compute measurements in distributed setups (in particular non-additive measurements). max_concurrent_cache_files (`int`): Max number of concurrent measurement cache files (default `10000`). timeout (`Union[int, float]`): Timeout in second for distributed setting synchronization. Nr4r)r)r)r*r7Jr5r7c@s<eZdZd ddZdddZdddZdd d Zd d ZdS)CombinedEvaluationsFcCsddlm}d|_t|tr||_nt|tr%t||_t||_g}|jD]}t|t r5||}| |q*||_|jdurLdd|jD|_||_ dS)Nr)loadcSsg|]}|jqSr))rU)r?moduler)r)r*r{z0CombinedEvaluations.__init__..) loadingr9evaluation_module_namesrrrKevaluation_modulesr rrrcr force_prefix)r#r>r?r9loaded_modulesr:r)r)r*r!is        zCombinedEvaluations.__init__Nc D|jD]}||d|fdd|D|jdiqdS)aYAdd one prediction and reference for each evaluation module's stack. Args: predictions (`list/array/tensor`, *optional*): Predictions. references (`list/array/tensor`, *optional*): References. Example: ```py >>> import evaluate >>> accuracy = evaluate.load("accuracy") >>> f1 = evaluate.load("f1") >>> clf_metrics = combine(["accuracy", "f1"]) >>> for ref, pred in zip([0,1,0,1], [1,0,0,1]): ... clf_metrics.add(references=ref, predictions=pred) ``` rcrr)r)rrr)r*rrz+CombinedEvaluations.add..Nr))r>rr)r#rrr&evaluation_moduler)rr*rs zCombinedEvaluations.addc rA)amAdd a batch of predictions and references for each evaluation module's stack. Args: predictions (`list/array/tensor`, *optional*): Predictions. references (`list/array/tensor`, *optional*): References. Example: ```py >>> import evaluate >>> accuracy = evaluate.load("accuracy") >>> f1 = evaluate.load("f1") >>> clf_metrics = combine(["accuracy", "f1"]) >>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): ... clf_metrics.add(references=refs, predictions=preds) ``` rcrr)r)rrr)r*rrz1CombinedEvaluations.add_batch..Nr))r>rr)r#rrr&rBr)rr*rs zCombinedEvaluations.add_batchcKs>g}|jD]}||d|}||jdi|q||S)aVCompute each evaluation module. Usage of positional arguments is not allowed to prevent mistakes. Args: predictions (`list/array/tensor`, *optional*): Predictions. references (`list/array/tensor`, *optional*): References. **kwargs (*optional*): Keyword arguments that will be forwarded to the evaluation module [`~evaluate.EvaluationModule.compute`] method (see details in the docstring). Return: `dict` or `None` - Dictionary with the results if this evaluation module is run on the main process (`process_id == 0`). - `None` if the evaluation module is not run on the main process (`process_id != 0`). Example: ```py >>> import evaluate >>> accuracy = evaluate.load("accuracy") >>> f1 = evaluate.load("f1") >>> clf_metrics = combine(["accuracy", "f1"]) >>> clf_metrics.compute(predictions=[0,1], references=[1,1]) {'accuracy': 0.5, 'f1': 0.6666666666666666} ``` rNr))r>rr_merge_results)r#rrr&resultsrBrr)r)r*rs   zCombinedEvaluations.computec Csi}ttjdd|D}ddt|D}ddt|jD}dd|D}t|j|D]B\}}|D]-\} } | |vrN|j sN| || <q=||vra| ||d||d| <q=| ||d| <q=||vrw||d 7<q5|S) NcSsg|]}|qSr))r)r?rr)r)r*rrz6CombinedEvaluations._merge_results..cSsh|] \}}|dkr|qSr/r)r?itemcountr)r)r* rz5CombinedEvaluations._merge_results..cSsg|] \}}|dkr|qSr/r)rFr)r)r*rscSsi|]}|dqSr r))r?rUr)r)r*rr;z6CombinedEvaluations._merge_results..rr) rK itertoolschain from_iterable collectionsCounterrr=rr?) r#rDmerged_results results_keysduplicate_keysduplicate_namesduplicate_counterrTresultrrr)r)r*rCs$ z"CombinedEvaluations._merge_resultsFr0)r6r7r8r!rrrrCr)r)r)r*r8hs      'r8FcCs t||dS)aCombines several metrics, comparisons, or measurements into a single `CombinedEvaluations` object that can be used like a single evaluation module. If two scores have the same name, then they are prefixed with their module names. And if two modules have the same name, please use a dictionary to give them different names, otherwise an integer id is appended to the prefix. Args: evaluations (`Union[list, dict]`): A list or dictionary of evaluation modules. The modules can either be passed as strings or loaded `EvaluationModule`s. If a dictionary is passed its keys are the names used and the values the modules. The names are used as prefix in case there are name overlaps in the returned results of each module or if `force_prefix=True`. force_prefix (`bool`, *optional*, defaults to `False`): If `True` all scores from the modules are prefixed with their name. If a dictionary is passed the keys are used as name otherwise the module's name. Examples: ```py >>> import evaluate >>> accuracy = evaluate.load("accuracy") >>> f1 = evaluate.load("f1") >>> clf_metrics = combine(["accuracy", "f1"]) ``` )r?)r8) evaluationsr?r)r)r*combines rWrU)sH          B}