evaluation-suite-ci.py

import evaluate
from evaluate.evaluation_suite import SubTask


class Suite(evaluate.EvaluationSuite):

    def __init__(self, name):
        super().__init__(name)

        self.preprocessor = lambda x: {"text": x["text"].lower()}
        self.suite = [
            SubTask(
                task_type="text-classification",
                data="imdb",
                split="test[:2]",
                data_preprocessor=self.preprocessor,
                args_for_task={
                    "metric": "accuracy",
                    "input_column": "sentence",
                    "label_column": "label",
                    "label_mapping": {"negative": 0, "positive": 1}
                }
            ),
            SubTask(
                task_type="text-classification",
                data="imdb",
                split="test[:2]",
                data_preprocessor=None,
                args_for_task={
                    "metric": "accuracy",
                    "input_column": "sentence",
                    "label_column": "label",
                    "label_mapping": {"negative": 0, "positive": 1}
                }
            )
        ]