Upload metrics.py with huggingface_hub

metrics.py

@@ -8,6 +8,7 @@ import evaluate
 import nltk
 import numpy
 
+from .dataclass import InternalField
 from .operator import (
     MultiStreamOperator,
     SingleStreamOperator,
@@ -60,7 +61,13 @@ class GlobalMetric(SingleStreamOperator, Metric):
 
             refs, pred = instance["references"], instance["prediction"]
 
-            instance_score = self._compute([refs], [pred])
+            try:
+                instance_score = self._compute([refs], [pred])
+            except:
+                instance_score = {"score": None}
+                if isinstance(self.main_score, str) and self.main_score is not None:
+                    instance_score[self.main_score] = None
+
             instance["score"]["instance"].update(instance_score)
 
             references.append(refs)
@@ -355,8 +362,27 @@ class Bleu(HuggingfaceMetric):
     scale = 1.0
 
 
+class MatthewsCorrelation(HuggingfaceMetric):
+    metric_name = "matthews_correlation"
+    main_score = "matthews_correlation"
+    str_to_id: dict = InternalField(default_factory=dict)
+
+    def get_str_id(self, str):
+        if str not in self.str_to_id:
+            id = len(self.str_to_id)
+            self.str_to_id[str] = id
+        return self.str_to_id[str]
+
+    def compute(self, references: List[List[str]], predictions: List[str]) -> dict:
+        formatted_references = [self.get_str_id(reference[0]) for reference in references]
+        formatted_predictions = [self.get_str_id(prediction) for prediction in predictions]
+        result = self.metric.compute(predictions=formatted_predictions, references=formatted_references)
+        return result
+
+
 class CustomF1(GlobalMetric):
     main_score = "f1_micro"
+    classes = None
 
     @abstractmethod
     def get_element_group(self, element):
@@ -391,6 +417,10 @@ class CustomF1(GlobalMetric):
         assert len(references) == len(predictions), (
             f"references size ({len(references)})" f" doesn't mach predictions sise ({len(references)})."
         )
+        if self.classes is None:
+            classes = set([self.get_element_group(e) for sublist in references for e in sublist])
+        else:
+            classes = self.classes
         groups_statistics = dict()
         for references_batch, predictions_batch in zip(references, predictions):
             grouped_references = self.group_elements(references_batch)
@@ -418,6 +448,7 @@ class CustomF1(GlobalMetric):
                 groups_statistics[group]["recall_denominator"] += rd
 
         result = {}
+        num_of_unknown_class_predictions = 0
         pn_total = pd_total = rn_total = rd_total = 0
         for group in groups_statistics.keys():
             pn, pd, rn, rd = (
@@ -426,13 +457,21 @@ class CustomF1(GlobalMetric):
                 groups_statistics[group]["recall_numerator"],
                 groups_statistics[group]["recall_denominator"],
             )
-            result[f"f1_{group}"] = self.f1(pn, pd, rn, rd)
             pn_total, pd_total, rn_total, rd_total = pn_total + pn, pd_total + pd, rn_total + rn, rd_total + rd
+            if group in classes:
+                result[f"f1_{group}"] = self.f1(pn, pd, rn, rd)
+            else:
+                num_of_unknown_class_predictions += pd
         try:
             result["f1_macro"] = sum(result.values()) / len(result.keys())
         except ZeroDivisionError:
             result["f1_macro"] = 1.0
 
+        amount_of_predictions = pd_total
+        if amount_of_predictions == 0:
+            result["in_classes_support"] = 1.0
+        else:
+            result["in_classes_support"] = 1.0 - num_of_unknown_class_predictions / amount_of_predictions
         result[f"f1_micro"] = self.f1(pn_total, pd_total, rn_total, rd_total)
         return result