o 5ti@sddlmZddlmZmZmZmZmZmZm Z ddl m Z m Z ddl mZddlmZddlmZmZdd lmZmZmZdd lmZerTdd lmZmZmZmZmZd Z Gd ddeZ!dS))Number) TYPE_CHECKINGAnyCallableDictOptionalTupleUnion)Dataset load_dataset)Literal)EvaluationModule)add_end_docstringsadd_start_docstrings)"EVALUATOR_COMPUTE_RETURN_DOCSTRING EVALUTOR_COMPUTE_START_DOCSTRING Evaluator)DatasetColumnPair)FeatureExtractionMixinPipelinePreTrainedModelPreTrainedTokenizerTFPreTrainedModelaD Examples: ```python >>> from evaluate import evaluator >>> from datasets import load_dataset >>> task_evaluator = evaluator("text-classification") >>> data = load_dataset("imdb", split="test[:2]") >>> results = task_evaluator.compute( >>> model_or_pipeline="huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli", >>> data=data, >>> metric="accuracy", >>> label_mapping={"LABEL_0": 0.0, "LABEL_1": 1.0}, >>> strategy="bootstrap", >>> n_resamples=10, >>> random_state=0 >>> ) ``` c(s<eZdZdZddiZd*fdd Zdeeefd ed ed efd d Z ddZ e e e ee                d+deededdfdeeefdeedeedeeefdeeedfdeeedfd ed!d"ed#ed$ed%eed ed eed ed&eeeefd'eeeefeff"d(d)ZZS),TextClassificationEvaluatora Text classification evaluator. This text classification evaluator can currently be loaded from [`evaluator`] using the default task name `text-classification` or with a `"sentiment-analysis"` alias. Methods in this class assume a data format compatible with the [`~transformers.TextClassificationPipeline`] - a single textual feature as input and a categorical label as output. truncationTtext-classificationNcstj||ddS)N)default_metric_name)super__init__)selftaskr __class__Z/home/ubuntu/.local/lib/python3.10/site-packages/evaluate/evaluator/text_classification.pyr >sz$TextClassificationEvaluator.__init__data input_columnsecond_input_column label_columncCsl|durtd||||d|dur||d|it|tr&t|n|}d||it|||ddfS)NzXPlease specify a valid `data` object - either a `str` with a name or a `Dataset` object.)r(r*r) referencestext text_pair) ValueErrorcheck_required_columns isinstancestrr r)r!r'r(r)r*r%r%r& prepare_dataAs  z(TextClassificationEvaluator.prepare_datacsfdd|D}d|iS)Ncs(g|]}dur|dn|dqS)Nlabelr%).0element label_mappingr%r& SszETextClassificationEvaluator.predictions_processor.. predictionsr%)r!r9r7r%r6r&predictions_processorRs z1TextClassificationEvaluator.predictions_processorsimpleffffff?'r,r3model_or_pipelinerrrsubsetsplitmetric tokenizerrfeature_extractorrstrategy)r; bootstrapconfidence_level n_resamplesdevice random_stater7returncCsi}|| ||j|||d}|j|| ||d\}}|j|||| d}||}|||\}}|||}|||j|||| | | d}|||||S)a# input_column (`str`, *optional*, defaults to `"text"`): The name of the column containing the text feature in the dataset specified by `data`. second_input_column (`str`, *optional*, defaults to `None`): The name of the second column containing the text features. This may be useful for classification tasks as MNLI, where two columns are used. label_column (`str`, defaults to `"label"`): The name of the column containing the labels in the dataset specified by `data`. label_mapping (`Dict[str, Number]`, *optional*, defaults to `None`): We want to map class labels defined by the model in the pipeline to values consistent with those defined in the `label_column` of the `data` dataset. )r'r?r@)r'r(r)r*)r>rBrCrH)rA metric_inputsrDrFrGrI) "check_for_mismatch_in_device_setup load_datar2prepare_pipelineprepare_metric call_pipeliner:updatecompute_metric)r!r>r'r?r@rArBrCrDrFrGrHrIr(r)r*r7resultrK pipe_inputspiper9 perf_resultsmetric_resultsr%r%r&computeYs6$      z#TextClassificationEvaluator.compute)rN)NNNNNNNr;r<r=NNr,Nr3N)__name__ __module__ __qualname____doc__PIPELINE_KWARGSr r r1r r2r:rrrrTASK_DOCUMENTATIONrrrr floatintrrrrrX __classcell__r%r%r#r&r3sz"        rN)"numbersrtypingrrrrrrr datasetsr r typing_extensionsr modulerutils.file_utilsrrbaserrrutilsr transformersrrrrrr^rr%r%r%r&s $