o 5ti%@sddlmZmZmZmZmZmZmZddlm Z ddl m Z ddl m Z ddlmZddlmZmZmZerBdd lmZmZmZmZd Zd Zd Zd ZGdddeZGdddeZGdddeZdS)) TYPE_CHECKINGAnyCallableDictOptionalTupleUnion)Dataset)Literal)EvaluationModule)add_start_docstrings)"EVALUATOR_COMPUTE_RETURN_DOCSTRING EVALUTOR_COMPUTE_START_DOCSTRING Evaluator)PipelinePreTrainedModelPreTrainedTokenizerTFPreTrainedModela input_column (`str`, defaults to `"text"`): the name of the column containing the input text in the dataset specified by `data`. label_column (`str`, defaults to `"label"`): the name of the column containing the labels in the dataset specified by `data`. generation_kwargs (`Dict`, *optional*, defaults to `None`): The generation kwargs are passed to the pipeline and set the text generation strategy. a Examples: ```python >>> from evaluate import evaluator >>> from datasets import load_dataset >>> task_evaluator = evaluator("text2text-generation") >>> data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]") >>> results = task_evaluator.compute( >>> model_or_pipeline="facebook/bart-large-cnn", >>> data=data, >>> input_column="article", >>> label_column="highlights", >>> metric="rouge", >>> ) ``` a Examples: ```python >>> from evaluate import evaluator >>> from datasets import load_dataset >>> task_evaluator = evaluator("summarization") >>> data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]") >>> results = task_evaluator.compute( >>> model_or_pipeline="facebook/bart-large-cnn", >>> data=data, >>> input_column="article", >>> label_column="highlights", >>> ) ``` a Examples: ```python >>> from evaluate import evaluator >>> from datasets import load_dataset >>> task_evaluator = evaluator("translation") >>> data = load_dataset("wmt19", "fr-de", split="validation[:40]") >>> data = data.map(lambda x: {"text": x["translation"]["de"], "label": x["translation"]["fr"]}) >>> results = task_evaluator.compute( >>> model_or_pipeline="Helsinki-NLP/opus-mt-de-fr", >>> data=data, >>> ) ``` c#seZdZdZdZddiZd&fdd Zd d Zee e e e            d'de ededdfde eefdeedeede eefdee edfdeddedededeed ed!ed"ed#eeeefefffd$d% ZZS)(Text2TextGenerationEvaluatora& Text2Text generation evaluator. This Text2Text generation evaluator can currently be loaded from [`evaluator`] using the default task name `text2text-generation`. Methods in this class assume a data format compatible with the [`~transformers.Text2TextGenerationPipeline`]. generated truncationTtext2text-generationNctj||ddSN)default_metric_namesuper__init__selftaskr __class__[/home/ubuntu/.local/lib/python3.10/site-packages/evaluate/evaluator/text2text_generation.pyrcz%Text2TextGenerationEvaluator.__init__csdfdd|DiS)N predictionscsg|] }|jdqS)_text)PREDICTION_PREFIX).0predr!r%r& gszFText2TextGenerationEvaluator.predictions_processor..r%)r!r( label_mappingr%r-r&predictions_processorfsz2Text2TextGenerationEvaluator.predictions_processorsimpleffffff?'textlabelmodel_or_pipelinerrrdatasubsetsplitmetric tokenizerrstrategyr1 bootstrapconfidence_level n_resamplesdevice random_state input_column label_columngeneration_kwargsreturncs>|dur |j|tj||||||||| | | | | d }|S)N) r6r7r8r9r:r;r<r?r@rArBrCrD)PIPELINE_KWARGSupdatercomputer!r6r7r8r9r:r;r<r?r@rArBrCrDrEresultr#r%r&rIis$ z$Text2TextGenerationEvaluator.compute)rNNNNNNNr1r2r3NNr4r5N)__name__ __module__ __qualname____doc__r*rGrr0r rTASK_DOCUMENTATION_KWARGSr TEXT2TEXT_TASK_DOCSTRING_EXAMPLErstrrr rr r floatintdictrrrrI __classcell__r%r%r#r&rXsv         rc#seZdZdZdZddiZd$fdd Zeee e e          d%de e deddfde e efdee dee de e efdee e dfdeddedededeede de d ed!eee efefffd"d# ZZS)&SummarizationEvaluatora Text summarization evaluator. This text summarization evaluator can currently be loaded from [`evaluator`] using the default task name `summarization`. Methods in this class assume a data format compatible with the [`SummarizationEvaluator`]. summaryrT summarizationNcrrrr r#r%r&rr'zSummarizationEvaluator.__init__r1r2r3r4r5r6rrrr7r8r9r:r;rr<r=r?r@rArBrCrDrErFc,tj||||||||| | | | | |d}|SN)r6r7r8r9r:r;r<r?r@rArBrCrDrErrIrJr#r%r&rI"zSummarizationEvaluator.compute)rZNrL)rMrNrOrPr*rGrr rrQr$SUMMARIZATION_TASK_DOCSTRING_EXAMPLErrSrr rr r rTrUrVrrrrIrWr%r%r#r&rXt         rXc#seZdZdZdZddiZd#fdd Zeee e e           d$d e e deddfde e efdee dee de e efdee e dfdeddedededeede de ded eee efefffd!d" ZZS)%TranslationEvaluatora Translation evaluator. This translation generation evaluator can currently be loaded from [`evaluator`] using the default task name `translation`. Methods in this class assume a data format compatible with the [`~transformers.TranslationPipeline`]. translationrTNcrrrr r#r%r&rr'zTranslationEvaluator.__init__r1r2r3r4r5r6rrrr7r8r9r:r;rr<r=r?r@rArBrCrDrErFcr[r\r]rJr#r%r&rIr^zTranslationEvaluator.compute)rbNrL)rMrNrOrPr*rGrr rrQr"TRANSLATION_TASK_DOCSTRING_EXAMPLErrSrr rr r rTrUrVrrrrIrWr%r%r#r&rar`raN) typingrrrrrrrdatasetsr typing_extensionsr moduler utils.file_utilsr baserrr transformersrrrrrQrRr_rcrrXrar%r%r%r&s$     @;